02:15:55.453 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:15:55.466 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskjrmL5D/test-tmp
02:15:55.469 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:15:55.474 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskjrmL5D/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:15:55.777 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:15:55.932 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:15:55.933 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:15:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:55.937891 30814 mini_oidc.cc:152] Starting JWKS server
02:15:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:55.942154 30814 webserver.cc:460] Webserver started at http://127.0.0.1:36735/ using document root <none> and password file <none>
02:15:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:55.942665 30814 mini_oidc.cc:189] Starting OIDC Discovery server
02:15:55.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:55.945230 30814 webserver.cc:460] Webserver started at http://127.0.0.1:45711/ using document root <none> and password file <none>
02:15:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:15:55Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:15:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:15:56Z Disabled control of system clock
02:15:56.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:56.021168 30814 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/wal
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:15:56.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/logs
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data/info.pb
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:15:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.190:38049
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.190
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:15:56.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:15:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:15:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:15:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:15:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:15:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:15:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36735/jwks.json with env {}
02:15:56.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.367764 30826 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:15:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.368422 30826 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:15:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.369201 30826 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:15:56.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.411816 30826 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:15:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.412168 30826 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:15:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.412374 30826 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:15:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.412564 30826 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:15:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.412762 30826 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:15:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.412978 30826 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36735/jwks.json
02:15:56.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:56.450959 30826 master_runner.cc:386] Master server non-default flags:
02:15:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:15:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:15:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:15:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:15:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data
02:15:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/wal
02:15:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:15:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.190:38049
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36735/jwks.json
02:15:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data/info.pb
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.190
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:15:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/logs
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:15:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:15:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:15:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:15:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:15:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:56.452612 30826 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:15:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:56.454708 30826 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:15:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.474922 30832 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:15:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.475714 30835 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:15:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:56.476953 30833 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:15:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:56.477020 30826 server_base.cc:1034] running on GCE node
02:15:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.618695 30826 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:15:57.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.622328 30826 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:15:57.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.623771 30826 hybrid_clock.cc:648] HybridClock initialized: now 1714529757623730 us; error 66 us; skew 500 ppm
02:15:57.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.624761 30826 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:15:57.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.636823 30826 webserver.cc:460] Webserver started at http://127.30.23.190:33897/ using document root <none> and password file <none>
02:15:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.637944 30826 fs_manager.cc:359] Metadata directory not provided
02:15:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.638177 30826 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:15:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.638649 30826 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:15:57.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.644332 30826 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data/instance:
02:15:57.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9aaa67e514ae48c3a430ab62990416f2"
02:15:57.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:15:57 on dist-test-slave-tmq0"
02:15:57.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.645728 30826 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/wal/instance:
02:15:57.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9aaa67e514ae48c3a430ab62990416f2"
02:15:57.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:15:57 on dist-test-slave-tmq0"
02:15:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.654002 30826 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
02:15:57.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.661027 30842 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:15:57.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.662495 30826 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
02:15:57.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.662978 30826 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/wal
02:15:57.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9aaa67e514ae48c3a430ab62990416f2"
02:15:57.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:15:57 on dist-test-slave-tmq0"
02:15:57.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.663493 30826 fs_report.cc:383] FS layout report
02:15:57.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:15:57.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/wal
02:15:57.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/wal
02:15:57.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data/data
02:15:57.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:15:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:15:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:15:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:15:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:15:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:15:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:15:57.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:15:57.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:15:57.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:15:57.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:15:57.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.741158 30826 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:15:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.743053 30826 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:15:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.743654 30826 kserver.cc:163] Server-wide thread pool size limit: 3276
02:15:57.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.827193 30826 rpc_server.cc:307] RPC server started. Bound to: 127.30.23.190:38049
02:15:57.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.827425 30893 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.23.190:38049 every 8 connection(s)
02:15:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.830516 30826 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/data/info.pb
02:15:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.835726 30814 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 30826
02:15:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.836264 30814 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-0/wal/instance
02:15:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.836661 30894 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:15:57.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.840057 30814 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:15:57.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:15:57.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/wal
02:15:57.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data
02:15:57.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/logs
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data/info.pb
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:15:57.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:15:57.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:15:57.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.189:35305
02:15:57.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.189
02:15:57.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:15:57.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:15:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:15:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:15:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:15:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:15:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:15:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:15:57.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36735/jwks.json with env {}
02:15:57.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.855520 30894 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } has no permanent_uuid. Determining permanent_uuid...
02:15:57.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.888671 30894 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } has no permanent_uuid. Determining permanent_uuid...
02:15:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:57.891651 30843 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:57.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:57.896550 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.189:35305: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:57.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:57.946234 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } attempt: 1
02:15:57.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:57.950906 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.189:35305: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:58.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.019486 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } attempt: 2
02:15:58.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.024452 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.189:35305: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.115943 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } attempt: 3
02:15:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.120637 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.189:35305: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:58.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.192836 30896 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:15:58.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.193537 30896 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:15:58.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.194132 30896 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:15:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.228834 30896 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:15:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.229233 30896 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:15:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.229449 30896 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:15:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.229650 30896 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:15:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.229846 30896 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:15:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.229981 30896 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36735/jwks.json
02:15:58.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.264124 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } attempt: 4
02:15:58.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.269132 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.189:35305: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.270690 30896 master_runner.cc:386] Master server non-default flags:
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/wal
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:15:58.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.189:35305
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36735/jwks.json
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data/info.pb
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.189
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:15:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:15:58.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/logs
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:15:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:15:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:15:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:15:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.272310 30896 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:15:58.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.274787 30896 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:15:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.295001 30908 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:15:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.568740 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } attempt: 5
02:15:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.575090 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.189:35305: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.122617 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } attempt: 6
02:15:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:59.130069 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.189:35305: Network error: Client connection negotiation failed: client connection to 127.30.23.189:35305: connect: Connection refused (error 111)
02:15:59.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.295030 30905 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:15:59.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:58.295730 30906 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:15:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:58.296345 30896 server_base.cc:1034] running on GCE node
02:15:59.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.433380 30896 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:15:59.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.436884 30896 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:15:59.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.438401 30896 hybrid_clock.cc:648] HybridClock initialized: now 1714529759438344 us; error 75 us; skew 500 ppm
02:15:59.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.439611 30896 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:15:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.448623 30896 webserver.cc:460] Webserver started at http://127.30.23.189:38435/ using document root <none> and password file <none>
02:15:59.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.450095 30896 fs_manager.cc:359] Metadata directory not provided
02:15:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.450726 30896 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:15:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.451472 30896 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:15:59.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.459228 30896 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data/instance:
02:15:59.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7"
02:15:59.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:15:59 on dist-test-slave-tmq0"
02:15:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.461071 30896 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/wal/instance:
02:15:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7"
02:15:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:15:59 on dist-test-slave-tmq0"
02:15:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.471813 30896 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.002s sys 0.009s
02:15:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.481751 30917 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:15:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.483554 30896 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.005s sys 0.000s
02:15:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.484064 30896 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/wal
02:15:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7"
02:15:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:15:59 on dist-test-slave-tmq0"
02:15:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.484505 30896 fs_report.cc:383] FS layout report
02:15:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:15:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/wal
02:15:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/wal
02:15:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data/data
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:15:59.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:15:59.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.540410 30896 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:15:59.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.542538 30896 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:15:59.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.543165 30896 kserver.cc:163] Server-wide thread pool size limit: 3276
02:15:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.618645 30896 rpc_server.cc:307] RPC server started. Bound to: 127.30.23.189:35305
02:15:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.618793 30968 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.23.189:35305 every 8 connection(s)
02:15:59.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.622257 30896 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/data/info.pb
02:15:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.629040 30969 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:15:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.630662 30814 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 30896
02:15:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.631155 30814 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-1/wal/instance
02:15:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.633888 30814 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:15:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:15:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/wal
02:15:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data
02:15:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:15:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:15:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/logs
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data/info.pb
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:15:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.188:44381
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.188
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:15:59.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36735/jwks.json with env {}
02:15:59.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.644433 30969 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } has no permanent_uuid. Determining permanent_uuid...
02:15:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.674614 30969 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } has no permanent_uuid. Determining permanent_uuid...
02:15:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.690382 30969 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } has no permanent_uuid. Determining permanent_uuid...
02:15:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:59.692849 30921 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:15:59.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:59.702421 30969 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:15:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.752157 30969 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 1
02:15:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:59.757650 30969 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:15:59.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.826262 30969 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 2
02:15:59.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:59.832675 30969 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:15:59.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:15:59.924165 30969 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 3
02:15:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:15:59.928382 30969 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.012465 30971 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.013108 30971 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.013783 30971 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.046967 30971 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:16:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.047336 30971 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.047588 30971 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:16:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.047782 30971 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:16:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.048138 30971 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:16:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.048381 30971 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36735/jwks.json
02:16:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.071995 30969 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 4
02:16:00.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.076349 30969 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.088045 30971 master_runner.cc:386] Master server non-default flags:
02:16:00.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:00.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:16:00.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/wal
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.188:44381
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36735/jwks.json
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data/info.pb
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.188
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/logs
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:00.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:00.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:00.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.089598 30971 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:00.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.091995 30971 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:00.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.110253 30981 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:00.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.191058 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } attempt: 7
02:16:00.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.218822 30894 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } has no permanent_uuid. Determining permanent_uuid...
02:16:00.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.226346 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.284799 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 1
02:16:00.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.290491 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.372119 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 2
02:16:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.375854 30969 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 5
02:16:00.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.378396 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.381024 30969 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.463889 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 3
02:16:00.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.470085 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.610605 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 4
02:16:00.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.615444 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.899441 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 5
02:16:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.904688 30894 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:00.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.928737 30969 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 6
02:16:00.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.937753 30969 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.23.188:44381: Network error: Client connection negotiation failed: client connection to 127.30.23.188:44381: connect: Connection refused (error 111)
02:16:01.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.110337 30984 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:01.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:00.111292 30982 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:01.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:00.111812 30971 server_base.cc:1034] running on GCE node
02:16:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.282415 30971 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:01.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.285506 30971 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:01.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.286918 30971 hybrid_clock.cc:648] HybridClock initialized: now 1714529761286864 us; error 60 us; skew 500 ppm
02:16:01.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.287945 30971 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:01.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.296133 30971 webserver.cc:460] Webserver started at http://127.30.23.188:37313/ using document root <none> and password file <none>
02:16:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.297279 30971 fs_manager.cc:359] Metadata directory not provided
02:16:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.297509 30971 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:01.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.297986 30971 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:16:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.304627 30971 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data/instance:
02:16:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a56b3209a3414370af940cd937574f7c"
02:16:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:01 on dist-test-slave-tmq0"
02:16:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.305922 30971 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/wal/instance:
02:16:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a56b3209a3414370af940cd937574f7c"
02:16:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:01 on dist-test-slave-tmq0"
02:16:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.315029 30971 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
02:16:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.321950 30996 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.323171 30971 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
02:16:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.323522 30971 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/wal
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a56b3209a3414370af940cd937574f7c"
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:01 on dist-test-slave-tmq0"
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.323868 30971 fs_report.cc:383] FS layout report
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/wal
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/wal
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data/data
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.373311 30971 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.375649 30971 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.376185 30971 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.453984 30971 rpc_server.cc:307] RPC server started. Bound to: 127.30.23.188:44381
02:16:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.454087 31047 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.23.188:44381 every 8 connection(s)
02:16:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.457201 30894 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 6
02:16:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.457859 30971 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/data/info.pb
02:16:01.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.466012 30814 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 30971
02:16:01.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.466598 30814 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/master-2/wal/instance
02:16:01.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.469249 31048 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:16:01.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.491089 31048 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } has no permanent_uuid. Determining permanent_uuid...
02:16:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.500932 30894 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2: Bootstrap starting.
02:16:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.510329 30894 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2: Neither blocks nor log segments found. Creating new log.
02:16:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.513607 30894 log.cc:824] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2: Log is configured to *not* fsync() on all Append() calls
02:16:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.521239 30894 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2: No bootstrap required, opened a new log
02:16:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.528829 31048 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } has no permanent_uuid. Determining permanent_uuid...
02:16:01.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.540292 31048 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } has no permanent_uuid. Determining permanent_uuid...
02:16:01.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.552572 30894 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:01.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.554179 30894 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:16:01.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.554524 30894 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9aaa67e514ae48c3a430ab62990416f2, State: Initialized, Role: FOLLOWER
02:16:01.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.555639 30894 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:01.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.566665 31048 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: Bootstrap starting.
02:16:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.574803 30894 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: configured and running, proceeding with master startup.
02:16:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.575452 31048 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: Neither blocks nor log segments found. Creating new log.
02:16:01.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.578085 31048 log.cc:824] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: Log is configured to *not* fsync() on all Append() calls
02:16:01.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.578649 31058 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:01.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.579998 31058 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: This master's current role is: FOLLOWER
02:16:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.586620 31048 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: No bootstrap required, opened a new log
02:16:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:01.608974 31070 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:01.609843 31070 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.626699 31048 raft_consensus.cc:348] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:01.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.628779 31048 raft_consensus.cc:374] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:16:01.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.629370 31048 raft_consensus.cc:729] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a56b3209a3414370af940cd937574f7c, State: Initialized, Role: FOLLOWER
02:16:01.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.630838 31048 consensus_queue.cc:260] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:01.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.654204 31048 sys_catalog.cc:564] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [sys.catalog]: configured and running, proceeding with master startup.
02:16:01.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.652082 31072 sys_catalog.cc:455] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.657243 31072 sys_catalog.cc:458] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [sys.catalog]: This master's current role is: FOLLOWER
02:16:01.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.607951 30826 master_runner.cc:186] Error getting master registration for 127.30.23.189:35305: OK, instance_id { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" instance_seqno: 1714529759595889 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:16:01.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.676717 30826 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:01.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:01.703966 31083 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:01.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:01.704799 31083 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.705224 30971 master_runner.cc:186] Error getting master registration for 127.30.23.189:35305: OK, instance_id { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" instance_seqno: 1714529759595889 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:16:01.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.744805 30971 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:01.755235 30848 tablet.cc:2369] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:16:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.873415 31058 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.874012 31058 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.878701 31058 leader_election.cc:288] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5325cb4bb44e4a00bcd9b0dd62e10ac7 (127.30.23.189:35305), a56b3209a3414370af940cd937574f7c (127.30.23.188:44381)
02:16:01.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.880574 31023 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9aaa67e514ae48c3a430ab62990416f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a56b3209a3414370af940cd937574f7c" is_pre_election: true
02:16:01.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.881629 31023 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9aaa67e514ae48c3a430ab62990416f2 in term 0.
02:16:01.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:01.882211 31002 tablet.cc:2369] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:16:01.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.883834 30846 leader_election.cc:302] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9aaa67e514ae48c3a430ab62990416f2, a56b3209a3414370af940cd937574f7c; no voters:
02:16:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.884846 31058 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:16:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.885380 31058 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:16:01.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.885720 31058 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 0 FOLLOWER]: Advancing to term 1
02:16:01.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.899566 31058 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.910943 31058 leader_election.cc:288] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [CANDIDATE]: Term 1 election: Requested vote from peers 5325cb4bb44e4a00bcd9b0dd62e10ac7 (127.30.23.189:35305), a56b3209a3414370af940cd937574f7c (127.30.23.188:44381)
02:16:01.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.911708 31023 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9aaa67e514ae48c3a430ab62990416f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a56b3209a3414370af940cd937574f7c"
02:16:01.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.912343 31023 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [term 0 FOLLOWER]: Advancing to term 1
02:16:01.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.920020 31023 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9aaa67e514ae48c3a430ab62990416f2 in term 1.
02:16:01.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.921469 30846 leader_election.cc:302] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9aaa67e514ae48c3a430ab62990416f2, a56b3209a3414370af940cd937574f7c; no voters:
02:16:01.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.924571 31058 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 1 FOLLOWER]: Leader election won for term 1
02:16:01.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.939500 31058 raft_consensus.cc:686] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [term 1 LEADER]: Becoming Leader. State: Replica: 9aaa67e514ae48c3a430ab62990416f2, State: Running, Role: LEADER
02:16:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.940697 31058 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:01.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.950690 31085 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9aaa67e514ae48c3a430ab62990416f2. Latest consensus state: current_term: 1 leader_uuid: "9aaa67e514ae48c3a430ab62990416f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.951776 31085 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: This master's current role is: LEADER
02:16:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.960776 31087 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:16:01.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.968446 31087 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:16:01.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.995066 31023 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [term 1 FOLLOWER]: Refusing update from remote peer 9aaa67e514ae48c3a430ab62990416f2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:16:01.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.997289 31058 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:16:01.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:01.998560 30969 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } attempt: 7
02:16:02.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.054028 31072 sys_catalog.cc:455] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9aaa67e514ae48c3a430ab62990416f2. Latest consensus state: current_term: 1 leader_uuid: "9aaa67e514ae48c3a430ab62990416f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:02.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.055306 31072 sys_catalog.cc:458] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [sys.catalog]: This master's current role is: FOLLOWER
02:16:02.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.061219 31088 mvcc.cc:205] Tried to move back new op lower bound from 7022713905103970304 to 7022713904920064000. Current Snapshot: MvccSnapshot[applied={T|T < 7022713905103970304}]
02:16:02.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.066597 31090 mvcc.cc:205] Tried to move back new op lower bound from 7022713905103970304 to 7022713904920064000. Current Snapshot: MvccSnapshot[applied={T|T < 7022713905103970304}]
02:16:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.074178 31085 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9aaa67e514ae48c3a430ab62990416f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.075008 31085 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: This master's current role is: LEADER
02:16:02.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.076910 31072 sys_catalog.cc:455] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9aaa67e514ae48c3a430ab62990416f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:02.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.077720 31072 sys_catalog.cc:458] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c [sys.catalog]: This master's current role is: FOLLOWER
02:16:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.082986 30969 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: Bootstrap starting.
02:16:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.083863 31087 catalog_manager.cc:1347] Generated new cluster ID: d816ffea40cf4ae689406ab9c4a5a211
02:16:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.084179 31087 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:16:02.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.094071 30969 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: Neither blocks nor log segments found. Creating new log.
02:16:02.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.115783 30969 log.cc:824] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: Log is configured to *not* fsync() on all Append() calls
02:16:02.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.122653 31087 catalog_manager.cc:1370] Generated new certificate authority record
02:16:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.130007 31087 catalog_manager.cc:1504] Loading token signing keys...
02:16:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.135699 30969 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: No bootstrap required, opened a new log
02:16:02.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.171890 30969 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:02.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.173388 30969 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:16:02.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.173872 30969 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5325cb4bb44e4a00bcd9b0dd62e10ac7, State: Initialized, Role: FOLLOWER
02:16:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.175091 30969 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } }
02:16:02.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.193336 31095 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:02.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.196058 31095 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [sys.catalog]: This master's current role is: FOLLOWER
02:16:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.197178 30943 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9aaa67e514ae48c3a430ab62990416f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7"
02:16:02.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.197494 30944 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9aaa67e514ae48c3a430ab62990416f2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" is_pre_election: true
02:16:02.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.198063 30969 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [sys.catalog]: configured and running, proceeding with master startup.
02:16:02.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.211834 30942 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [term 0 FOLLOWER]: Advancing to term 1
02:16:02.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.237671 30942 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [term 1 FOLLOWER]: Refusing update from remote peer 9aaa67e514ae48c3a430ab62990416f2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:16:02.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.240000 31085 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:16:02.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.264958 31087 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2: Generated new TSK 0
02:16:02.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.266647 31087 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:16:02.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:02.303427 30923 tablet.cc:2369] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:16:02.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.313184 30814 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/wal
02:16:02.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/logs
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data/info.pb
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:16:02.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.129:0
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.23.129
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.129
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:02.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:16:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.342607 31058 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9aaa67e514ae48c3a430ab62990416f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.343508 31058 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9aaa67e514ae48c3a430ab62990416f2 [sys.catalog]: This master's current role is: LEADER
02:16:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.345198 31095 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9aaa67e514ae48c3a430ab62990416f2. Latest consensus state: current_term: 1 leader_uuid: "9aaa67e514ae48c3a430ab62990416f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:02.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.345991 31095 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [sys.catalog]: This master's current role is: FOLLOWER
02:16:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.347025 31108 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9aaa67e514ae48c3a430ab62990416f2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9aaa67e514ae48c3a430ab62990416f2" member_type: VOTER last_known_addr { host: "127.30.23.190" port: 38049 } } peers { permanent_uuid: "5325cb4bb44e4a00bcd9b0dd62e10ac7" member_type: VOTER last_known_addr { host: "127.30.23.189" port: 35305 } } peers { permanent_uuid: "a56b3209a3414370af940cd937574f7c" member_type: VOTER last_known_addr { host: "127.30.23.188" port: 44381 } } }
02:16:02.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.348884 31108 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7 [sys.catalog]: This master's current role is: FOLLOWER
02:16:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.386010 31112 catalog_manager.cc:1259] Loaded cluster ID: d816ffea40cf4ae689406ab9c4a5a211
02:16:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.386376 31112 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: loading cluster ID for follower catalog manager: success
02:16:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.401294 31112 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: acquiring CA information for follower catalog manager: success
02:16:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.407239 31112 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5325cb4bb44e4a00bcd9b0dd62e10ac7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:16:02.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.711472 31083 catalog_manager.cc:1259] Loaded cluster ID: d816ffea40cf4ae689406ab9c4a5a211
02:16:02.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.711817 31083 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: loading cluster ID for follower catalog manager: success
02:16:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.734002 31083 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: acquiring CA information for follower catalog manager: success
02:16:02.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:02.777309 31083 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a56b3209a3414370af940cd937574f7c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:16:02.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:02.939877 31109 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:02.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:02.940629 31109 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:02.941462 31109 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:03.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:03.004206 31109 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:03.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:03.005714 31109 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.23.129
02:16:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:03.071163 31109 tablet_server_runner.cc:78] Tablet server non-default flags:
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/wal
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.129:0
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data/info.pb
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.129
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.23.129
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/logs
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:16:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:03.073092 31109 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:03.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:03.075609 31109 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:03.122027 31120 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:04.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:03.135251 31109 server_base.cc:1034] running on GCE node
02:16:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:03.135908 31124 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:03.129742 31121 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:04.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.583017 31109 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:04.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.594578 31109 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.596319 31109 hybrid_clock.cc:648] HybridClock initialized: now 1714529764596260 us; error 79 us; skew 500 ppm
02:16:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.597604 31109 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.619277 31109 webserver.cc:460] Webserver started at http://127.30.23.129:45435/ using document root <none> and password file <none>
02:16:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.620792 31109 fs_manager.cc:359] Metadata directory not provided
02:16:04.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.621126 31109 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.621758 31109 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:16:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.630061 31109 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data/instance:
02:16:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "653c0c72c11f4354a78958a7188c5a88"
02:16:04.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:04 on dist-test-slave-tmq0"
02:16:04.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.631872 31109 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/wal/instance:
02:16:04.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "653c0c72c11f4354a78958a7188c5a88"
02:16:04.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:04 on dist-test-slave-tmq0"
02:16:04.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.649037 31109 fs_manager.cc:691] Time spent creating directory manager: real 0.016s user 0.013s sys 0.002s
02:16:04.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.663954 31133 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:04.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.665676 31109 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.000s sys 0.008s
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.666170 31109 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/wal
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "653c0c72c11f4354a78958a7188c5a88"
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:04 on dist-test-slave-tmq0"
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.666695 31109 fs_report.cc:383] FS layout report
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/wal
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/wal
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data/data
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.783814 31109 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:04.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.786185 31109 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:04.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.786907 31109 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:04.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.790818 31109 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:16:04.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.797482 31109 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:16:04.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.797830 31109 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:16:04.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.798166 31109 ts_tablet_manager.cc:590] Registered 0 tablets
02:16:04.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:04.798395 31109 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:16:05.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.031633 31109 rpc_server.cc:307] RPC server started. Bound to: 127.30.23.129:43651
02:16:05.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.031774 31245 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.23.129:43651 every 8 connection(s)
02:16:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.035593 31109 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/data/info.pb
02:16:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.046833 30814 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31109
02:16:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.047405 30814 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-0/wal/instance
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.070806 30814 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/wal
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/logs
02:16:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data/info.pb
02:16:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.130:0
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.23.130
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.130
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:16:05.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.115017 31246 heartbeater.cc:347] Connected to a master server at 127.30.23.188:44381
02:16:05.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.115613 31246 heartbeater.cc:464] Registering TS with master...
02:16:05.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.117044 31246 heartbeater.cc:511] Master 127.30.23.188:44381 requested a full tablet report, sending...
02:16:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.118579 31247 heartbeater.cc:347] Connected to a master server at 127.30.23.189:35305
02:16:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.118945 31247 heartbeater.cc:464] Registering TS with master...
02:16:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.119902 31247 heartbeater.cc:511] Master 127.30.23.189:35305 requested a full tablet report, sending...
02:16:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.122995 31013 ts_manager.cc:194] Registered new tserver with Master: 653c0c72c11f4354a78958a7188c5a88 (127.30.23.129:43651)
02:16:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.123219 30934 ts_manager.cc:194] Registered new tserver with Master: 653c0c72c11f4354a78958a7188c5a88 (127.30.23.129:43651)
02:16:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.140452 31252 heartbeater.cc:347] Connected to a master server at 127.30.23.190:38049
02:16:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.140836 31252 heartbeater.cc:464] Registering TS with master...
02:16:05.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.141836 31252 heartbeater.cc:511] Master 127.30.23.190:38049 requested a full tablet report, sending...
02:16:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.144600 30859 ts_manager.cc:194] Registered new tserver with Master: 653c0c72c11f4354a78958a7188c5a88 (127.30.23.129:43651)
02:16:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.147589 30859 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.23.129:47101
02:16:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.484593 31254 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.485327 31254 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:05.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.486104 31254 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.526142 31254 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:05.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.527223 31254 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.23.130
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.565465 31254 tablet_server_runner.cc:78] Tablet server non-default flags:
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/wal
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.130:0
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data/info.pb
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.130
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.23.130
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/logs
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:05.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:05.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.567030 31254 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:05.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.569346 31254 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.592238 31262 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.152719 31252 heartbeater.cc:503] Master 127.30.23.190:38049 was elected leader, sending a full tablet report...
02:16:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.592288 31265 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:06.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:05.592960 31263 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:06.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:05.594173 31254 server_base.cc:1034] running on GCE node
02:16:06.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.769223 31254 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.772527 31254 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.774060 31254 hybrid_clock.cc:648] HybridClock initialized: now 1714529766773985 us; error 85 us; skew 500 ppm
02:16:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.775290 31254 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:06.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.784442 31254 webserver.cc:460] Webserver started at http://127.30.23.130:43071/ using document root <none> and password file <none>
02:16:06.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.785615 31254 fs_manager.cc:359] Metadata directory not provided
02:16:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.785871 31254 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.786458 31254 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:16:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.792483 31254 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data/instance:
02:16:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af5704b58514491c8cf83c2ac66c15b1"
02:16:06.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:06 on dist-test-slave-tmq0"
02:16:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.794091 31254 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/wal/instance:
02:16:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af5704b58514491c8cf83c2ac66c15b1"
02:16:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:06 on dist-test-slave-tmq0"
02:16:06.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.802434 31254 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
02:16:06.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.808977 31274 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:06.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.810576 31254 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:16:06.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.811079 31254 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/wal
02:16:06.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af5704b58514491c8cf83c2ac66c15b1"
02:16:06.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:06 on dist-test-slave-tmq0"
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.811599 31254 fs_report.cc:383] FS layout report
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/wal
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/wal
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data/data
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:06.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:06.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.871965 31254 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:06.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.873711 31254 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:06.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.874290 31254 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.877490 31254 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:16:06.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.882300 31254 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:16:06.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.882547 31254 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:16:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.882849 31254 ts_tablet_manager.cc:590] Registered 0 tablets
02:16:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:06.883045 31254 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:16:07.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.026773 31254 rpc_server.cc:307] RPC server started. Bound to: 127.30.23.130:45933
02:16:07.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.026888 31386 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.23.130:45933 every 8 connection(s)
02:16:07.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.029641 31254 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/data/info.pb
02:16:07.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.037456 30814 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31254
02:16:07.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.038292 30814 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-1/wal/instance
02:16:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.056317 30814 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/wal
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/logs
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data/info.pb
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:16:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.131:0
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.23.131
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.131
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:07.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:16:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.094650 31391 heartbeater.cc:347] Connected to a master server at 127.30.23.190:38049
02:16:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.095227 31391 heartbeater.cc:464] Registering TS with master...
02:16:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.096436 31387 heartbeater.cc:347] Connected to a master server at 127.30.23.188:44381
02:16:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.096570 31391 heartbeater.cc:511] Master 127.30.23.190:38049 requested a full tablet report, sending...
02:16:07.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.096840 31387 heartbeater.cc:464] Registering TS with master...
02:16:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.097808 31387 heartbeater.cc:511] Master 127.30.23.188:44381 requested a full tablet report, sending...
02:16:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.100769 31388 heartbeater.cc:347] Connected to a master server at 127.30.23.189:35305
02:16:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.100965 31013 ts_manager.cc:194] Registered new tserver with Master: af5704b58514491c8cf83c2ac66c15b1 (127.30.23.130:45933)
02:16:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.101161 31388 heartbeater.cc:464] Registering TS with master...
02:16:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.101433 30859 ts_manager.cc:194] Registered new tserver with Master: af5704b58514491c8cf83c2ac66c15b1 (127.30.23.130:45933)
02:16:07.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.102090 31388 heartbeater.cc:511] Master 127.30.23.189:35305 requested a full tablet report, sending...
02:16:07.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.103293 30859 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.23.130:53801
02:16:07.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.104967 30934 ts_manager.cc:194] Registered new tserver with Master: af5704b58514491c8cf83c2ac66c15b1 (127.30.23.130:45933)
02:16:07.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.409766 31395 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:07.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.410334 31395 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:07.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.410809 31395 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.444407 31395 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:07.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.445286 31395 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.23.131
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.485086 31395 tablet_server_runner.cc:78] Tablet server non-default flags:
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.23.148:38491
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/wal
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.23.131:0
02:16:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data/info.pb
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.23.131
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.23.190:38049,127.30.23.189:35305,127.30.23.188:44381
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.23.131
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/logs
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.486511 31395 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.488618 31395 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.507477 31403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:08.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.107954 31391 heartbeater.cc:503] Master 127.30.23.190:38049 was elected leader, sending a full tablet report...
02:16:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.508800 31406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:07.510313 31395 server_base.cc:1034] running on GCE node
02:16:08.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:07.508951 31404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.666323 31395 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:08.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.668761 31395 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:08.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.670156 31395 hybrid_clock.cc:648] HybridClock initialized: now 1714529768670100 us; error 80 us; skew 500 ppm
02:16:08.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.670967 31395 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:08.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.677467 31395 webserver.cc:460] Webserver started at http://127.30.23.131:38481/ using document root <none> and password file <none>
02:16:08.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.678396 31395 fs_manager.cc:359] Metadata directory not provided
02:16:08.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.678594 31395 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:08.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.679003 31395 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:16:08.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.684157 31395 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data/instance:
02:16:08.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eba6c1b81f21421684163c5667a658bb"
02:16:08.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:08 on dist-test-slave-tmq0"
02:16:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.685302 31395 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/wal/instance:
02:16:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eba6c1b81f21421684163c5667a658bb"
02:16:08.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:08 on dist-test-slave-tmq0"
02:16:08.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.692643 31395 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
02:16:08.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.698179 31415 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:08.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.699404 31395 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.699759 31395 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/wal
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eba6c1b81f21421684163c5667a658bb"
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:08 on dist-test-slave-tmq0"
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.700220 31395 fs_report.cc:383] FS layout report
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/wal
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/wal
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data/data
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:08.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:08.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.772804 31395 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:08.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.774667 31395 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:08.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.775221 31395 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:08.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.777931 31395 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:16:08.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.782431 31395 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:16:08.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.782655 31395 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:16:08.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.782976 31395 ts_tablet_manager.cc:590] Registered 0 tablets
02:16:08.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.783161 31395 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:16:08.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.945871 31395 rpc_server.cc:307] RPC server started. Bound to: 127.30.23.131:41761
02:16:08.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.945993 31527 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.23.131:41761 every 8 connection(s)
02:16:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.948649 31395 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/data/info.pb
02:16:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.956169 30814 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31395
02:16:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.956584 30814 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster6793721429708526761/ts-2/wal/instance
02:16:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.992589 31530 heartbeater.cc:347] Connected to a master server at 127.30.23.190:38049
02:16:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.992866 31528 heartbeater.cc:347] Connected to a master server at 127.30.23.188:44381
02:16:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.993136 31530 heartbeater.cc:464] Registering TS with master...
02:16:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.993208 31528 heartbeater.cc:464] Registering TS with master...
02:16:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.994500 31530 heartbeater.cc:511] Master 127.30.23.190:38049 requested a full tablet report, sending...
02:16:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.994571 31528 heartbeater.cc:511] Master 127.30.23.188:44381 requested a full tablet report, sending...
02:16:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.995867 31529 heartbeater.cc:347] Connected to a master server at 127.30.23.189:35305
02:16:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.996177 31529 heartbeater.cc:464] Registering TS with master...
02:16:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.996910 31529 heartbeater.cc:511] Master 127.30.23.189:35305 requested a full tablet report, sending...
02:16:08.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.997853 31013 ts_manager.cc:194] Registered new tserver with Master: eba6c1b81f21421684163c5667a658bb (127.30.23.131:41761)
02:16:08.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:08.999264 30859 ts_manager.cc:194] Registered new tserver with Master: eba6c1b81f21421684163c5667a658bb (127.30.23.131:41761)
02:16:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:09.000476 30934 ts_manager.cc:194] Registered new tserver with Master: eba6c1b81f21421684163c5667a658bb (127.30.23.131:41761)
02:16:09.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:09.000967 30859 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.23.131:34441
02:16:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:09.009871 30814 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
02:16:09.021 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:16:09.022 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:16:09.038 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.23.190"
port: 38049
}
bound_http_address {
host: "127.30.23.190"
port: 33897
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.23.189"
port: 35305
}
bound_http_address {
host: "127.30.23.189"
port: 38435
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.23.188"
port: 44381
}
bound_http_address {
host: "127.30.23.188"
port: 37313
}
}
}
02:16:09.044 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:16:09.047 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.30.23.129"
port: 43651
}
bound_http_address {
host: "127.30.23.129"
port: 45435
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.30.23.130"
port: 45933
}
bound_http_address {
host: "127.30.23.130"
port: 43071
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.30.23.131"
port: 41761
}
bound_http_address {
host: "127.30.23.131"
port: 38481
}
}
}
02:16:09.047 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:16:09.224 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:16:09.354 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskjrmL5D/test-tmp
02:16:09.355 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:16:09.355 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskjrmL5D/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:16:09.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:16:09.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:09.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:16:09.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:09.719851 31539 mini_oidc.cc:152] Starting JWKS server
02:16:09.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:09.723510 31539 webserver.cc:460] Webserver started at http://127.0.0.1:44721/ using document root <none> and password file <none>
02:16:09.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:09.724126 31539 mini_oidc.cc:189] Starting OIDC Discovery server
02:16:09.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:09.726364 31539 webserver.cc:460] Webserver started at http://127.0.0.1:43553/ using document root <none> and password file <none>
02:16:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:16:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/principal' for realm 'KRBTEST.COM',
02:16:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:16:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:09 dist-test-slave-tmq0 krb5kdc[31551](info): setting up network...
02:16:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:16:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:09 dist-test-slave-tmq0 krb5kdc[31551](info): set up 2 sockets
02:16:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:09 dist-test-slave-tmq0 krb5kdc[31551](info): commencing operation
02:16:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:16:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:10.004377 31530 heartbeater.cc:503] Master 127.30.23.190:38049 was elected leader, sending a full tablet report...
02:16:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:10.766234 31539 mini_kdc.cc:121] Time spent starting KDC: real 0.991s user 0.002s sys 0.021s
02:16:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:16:10.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:16:10.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:16:10.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:16:10.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:16:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:16:10.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:16:10.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:16:10.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:16:10.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:16:10.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/test-user.keytab.
02:16:10.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/test-user.keytab.
02:16:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:10 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:16:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:16:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:16:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:16:10Z Disabled control of system clock
02:16:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.204.254@KRBTEST.COM; defaulting to no policy
02:16:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:16:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.204.254@KRBTEST.COM" created.
02:16:10.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:16:10.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.204.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.254.keytab.
02:16:10.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.204.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.254.keytab.
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:10.988801 31539 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/logs
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/info.pb
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.254:33425
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.254
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.254.keytab
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.254
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:10.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:16:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.332578 31567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.333190 31567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:11.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.333777 31567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.367405 31567 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:16:11.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.367718 31567 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:16:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.368016 31567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.368228 31567 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:16:11.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.368472 31567 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:16:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.368819 31567 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:16:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.369100 31567 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44721/jwks.json
02:16:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:11.408213 31567 master_runner.cc:386] Master server non-default flags:
02:16:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.254.keytab
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.254
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.254:33425
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/info.pb
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.254
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/logs
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:16:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:11.409746 31567 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:11.410873 31567 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.428362 31573 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.429209 31574 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:11.432490 31567 server_base.cc:1034] running on GCE node
02:16:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:11.429682 31576 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.593355 31567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.596239 31567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.597656 31567 hybrid_clock.cc:648] HybridClock initialized: now 1714529772597608 us; error 72 us; skew 500 ppm
02:16:12.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.598623 31567 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:12.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:12 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529772, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:12.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.605355 31567 init.cc:377] Logged in from keytab as kudu/127.30.204.254@KRBTEST.COM (short username kudu)
02:16:12.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.611403 31567 webserver.cc:460] Webserver started at http://127.30.204.254:38703/ using document root <none> and password file <none>
02:16:12.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.612397 31567 fs_manager.cc:359] Metadata directory not provided
02:16:12.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.612607 31567 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:12.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.613044 31567 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:16:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.618873 31567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/instance:
02:16:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae"
02:16:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:12 on dist-test-slave-tmq0"
02:16:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.620548 31567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal/instance:
02:16:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae"
02:16:12.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:12 on dist-test-slave-tmq0"
02:16:12.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.632076 31567 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.006s sys 0.005s
02:16:12.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.640056 31584 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.641476 31567 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
02:16:12.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.641903 31567 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:12.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae"
02:16:12.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:12 on dist-test-slave-tmq0"
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.642370 31567 fs_report.cc:383] FS layout report
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/data
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.705528 31567 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.711508 31567 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.711959 31567 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:12.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.802839 31567 rpc_server.cc:307] RPC server started. Bound to: 127.30.204.254:33425
02:16:12.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.802954 31635 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.204.254:33425 every 8 connection(s)
02:16:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.805920 31567 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/info.pb
02:16:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.812423 31636 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:16:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.814035 31539 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31567
02:16:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.814443 31539 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal/instance
02:16:12.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.828344 31636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } has no permanent_uuid. Determining permanent_uuid...
02:16:12.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.204.253@KRBTEST.COM; defaulting to no policy
02:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:16:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.204.253@KRBTEST.COM" created.
02:16:12.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:12 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529772, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.254@KRBTEST.COM for kudu/127.30.204.254@KRBTEST.COM
02:16:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:16:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.204.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.253.keytab.
02:16:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.204.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.253.keytab.
02:16:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.877858 31636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } has no permanent_uuid. Determining permanent_uuid...
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.879778 31539 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/logs
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/info.pb
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.253:37985
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.253
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.253.keytab
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.253
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:16:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:12.880774 31588 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:12.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:12.885380 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.253:37985: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:12.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:12.935045 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } attempt: 1
02:16:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:12.939422 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.253:37985: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.007824 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } attempt: 2
02:16:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.012466 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.253:37985: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.103844 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } attempt: 3
02:16:13.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.108639 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.253:37985: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:13.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.231482 31644 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:13.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.232168 31644 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:13.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.232780 31644 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.252070 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } attempt: 4
02:16:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.256294 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.253:37985: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:13.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.267254 31644 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:16:13.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.267589 31644 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:16:13.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.268008 31644 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:13.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.268332 31644 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:16:13.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.268678 31644 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:16:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.268996 31644 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:16:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.269218 31644 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44721/jwks.json
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.309084 31644 master_runner.cc:386] Master server non-default flags:
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.253.keytab
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.253
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.253:37985
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/info.pb
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.253
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/logs
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.310776 31644 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.312103 31644 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:13.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.331421 31650 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:13.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.555723 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } attempt: 5
02:16:13.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.560423 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.253:37985: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.107841 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } attempt: 6
02:16:14.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:14.116048 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.253:37985: Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.332043 31653 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:13.332362 31651 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:14.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:13.333199 31644 server_base.cc:1034] running on GCE node
02:16:14.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.518436 31644 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:14.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.521867 31644 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:14.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.523391 31644 hybrid_clock.cc:648] HybridClock initialized: now 1714529774523354 us; error 50 us; skew 500 ppm
02:16:14.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.524248 31644 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:14.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:14 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529774, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.530331 31644 init.cc:377] Logged in from keytab as kudu/127.30.204.253@KRBTEST.COM (short username kudu)
02:16:14.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.536319 31644 webserver.cc:460] Webserver started at http://127.30.204.253:35677/ using document root <none> and password file <none>
02:16:14.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.537379 31644 fs_manager.cc:359] Metadata directory not provided
02:16:14.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.537645 31644 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.538192 31644 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:16:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.544113 31644 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/instance:
02:16:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c71067590384d838bbf8f09fec01268"
02:16:14.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:14 on dist-test-slave-tmq0"
02:16:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.545454 31644 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal/instance:
02:16:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c71067590384d838bbf8f09fec01268"
02:16:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:14 on dist-test-slave-tmq0"
02:16:14.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.553409 31644 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
02:16:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.559827 31663 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.561074 31644 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:16:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.561393 31644 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c71067590384d838bbf8f09fec01268"
02:16:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:14 on dist-test-slave-tmq0"
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.561715 31644 fs_report.cc:383] FS layout report
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/data
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:14.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.614074 31644 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:14.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.620388 31644 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.620954 31644 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.704546 31644 rpc_server.cc:307] RPC server started. Bound to: 127.30.204.253:37985
02:16:14.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.704612 31714 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.204.253:37985 every 8 connection(s)
02:16:14.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.707489 31644 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/info.pb
02:16:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.710716 31539 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31644
02:16:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.711374 31539 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal/instance
02:16:14.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.714679 31715 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:16:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.204.252@KRBTEST.COM; defaulting to no policy
02:16:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.728739 31715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } has no permanent_uuid. Determining permanent_uuid...
02:16:14.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:16:14.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.204.252@KRBTEST.COM" created.
02:16:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:14 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529774, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.253@KRBTEST.COM for kudu/127.30.204.254@KRBTEST.COM
02:16:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:16:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.204.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.252.keytab.
02:16:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.204.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.252.keytab.
02:16:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.763572 31539 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/logs
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/info.pb
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.252:37931
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.252
02:16:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.252.keytab
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.252
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:16:14.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.775190 31715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } has no permanent_uuid. Determining permanent_uuid...
02:16:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:14 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529774, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.253@KRBTEST.COM for kudu/127.30.204.253@KRBTEST.COM
02:16:14.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.806816 31715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } has no permanent_uuid. Determining permanent_uuid...
02:16:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:14.810201 31666 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:14.815842 31715 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.865453 31715 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 1
02:16:14.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:14.870183 31715 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:14.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:14.938589 31715 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 2
02:16:14.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:14.942658 31715 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.034117 31715 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 3
02:16:15.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.040400 31715 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.130349 31721 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.131668 31721 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.132385 31721 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.168859 31721 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:16:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.169149 31721 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:16:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.169577 31721 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.169910 31721 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:16:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.170224 31721 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:16:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.170536 31721 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:16:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.170768 31721 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44721/jwks.json
02:16:15.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.176721 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } attempt: 7
02:16:15.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.183887 31715 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 4
02:16:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.188601 31715 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:15 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529772, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.254@KRBTEST.COM for kudu/127.30.204.253@KRBTEST.COM
02:16:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.207170 31636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } has no permanent_uuid. Determining permanent_uuid...
02:16:15.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.214272 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.222486 31721 master_runner.cc:386] Master server non-default flags:
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.252.keytab
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.252
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.252:37931
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:16:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/info.pb
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.252
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/logs
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.223838 31721 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:15.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.224925 31721 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:15.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.243796 31731 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:15.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.272738 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 1
02:16:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.276849 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.358562 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 2
02:16:15.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.363449 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.449332 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 3
02:16:15.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.456912 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.488059 31715 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 5
02:16:15.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.492727 31715 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.597358 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 4
02:16:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.601758 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.885499 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 5
02:16:15.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.890096 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.040263 31715 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 6
02:16:16.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:16.047497 31715 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:16.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.442884 31636 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } attempt: 6
02:16:16.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:16.452242 31636 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.204.252:37931: Network error: Client connection negotiation failed: client connection to 127.30.204.252:37931: connect: Connection refused (error 111)
02:16:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.244699 31734 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:15.245009 31721 server_base.cc:1034] running on GCE node
02:16:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:15.244452 31732 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.457335 31721 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:16.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.460991 31721 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.462702 31721 hybrid_clock.cc:648] HybridClock initialized: now 1714529776462645 us; error 58 us; skew 500 ppm
02:16:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.463867 31721 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:16.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529776, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.472260 31721 init.cc:377] Logged in from keytab as kudu/127.30.204.252@KRBTEST.COM (short username kudu)
02:16:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.478652 31721 webserver.cc:460] Webserver started at http://127.30.204.252:46195/ using document root <none> and password file <none>
02:16:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.480015 31721 fs_manager.cc:359] Metadata directory not provided
02:16:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.480257 31721 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.480701 31721 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:16:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.486158 31721 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/instance:
02:16:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82e719f776d648f5af481807d2efb29b"
02:16:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:16 on dist-test-slave-tmq0"
02:16:16.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.487437 31721 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal/instance:
02:16:16.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82e719f776d648f5af481807d2efb29b"
02:16:16.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:16 on dist-test-slave-tmq0"
02:16:16.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.495618 31721 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
02:16:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.501906 31749 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.503101 31721 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
02:16:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.503518 31721 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82e719f776d648f5af481807d2efb29b"
02:16:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:16 on dist-test-slave-tmq0"
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.503867 31721 fs_report.cc:383] FS layout report
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/data
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:16.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.559947 31721 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.565867 31721 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.566401 31721 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:16.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.663929 31721 rpc_server.cc:307] RPC server started. Bound to: 127.30.204.252:37931
02:16:16.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.663970 31800 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.204.252:37931 every 8 connection(s)
02:16:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.667919 31721 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/info.pb
02:16:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.674604 31539 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31721
02:16:16.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.675120 31539 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal/instance
02:16:16.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.675933 31801 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:16:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.696030 31801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } has no permanent_uuid. Determining permanent_uuid...
02:16:16.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.204.252@KRBTEST.COM
02:16:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529776, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for kudu/127.30.204.254@KRBTEST.COM
02:16:16.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.749219 31801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } has no permanent_uuid. Determining permanent_uuid...
02:16:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.204.253@KRBTEST.COM
02:16:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529776, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for kudu/127.30.204.253@KRBTEST.COM
02:16:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.777223 31801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } has no permanent_uuid. Determining permanent_uuid...
02:16:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.204.254@KRBTEST.COM
02:16:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529776, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for kudu/127.30.204.252@KRBTEST.COM
02:16:16.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.802910 31539 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
02:16:16.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:16.814 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:16:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.814121 31801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Bootstrap starting.
02:16:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.822502 31801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Neither blocks nor log segments found. Creating new log.
02:16:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.825583 31801 log.cc:824] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Log is configured to *not* fsync() on all Append() calls
02:16:16.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:16:16.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:16:16.836 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.204.254"
port: 33425
}
bound_http_address {
host: "127.30.204.254"
port: 38703
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.204.253"
port: 37985
}
bound_http_address {
host: "127.30.204.253"
port: 35677
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.204.252"
port: 37931
}
bound_http_address {
host: "127.30.204.252"
port: 46195
}
}
}
02:16:16.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:16:16.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.837762 31801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: No bootstrap required, opened a new log
02:16:16.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:16.838 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:16:16.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:16:16.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.870563 31801 raft_consensus.cc:348] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:16.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.871647 31801 raft_consensus.cc:374] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:16:16.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.872035 31801 raft_consensus.cc:729] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82e719f776d648f5af481807d2efb29b, State: Initialized, Role: FOLLOWER
02:16:16.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.873116 31801 consensus_queue.cc:260] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:16.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529776, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:16.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:16:16.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.876329 31812 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } } }
02:16:16.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.877207 31812 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [sys.catalog]: This master's current role is: FOLLOWER
02:16:16.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.878501 31801 sys_catalog.cc:564] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [sys.catalog]: configured and running, proceeding with master startup.
02:16:16.885 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:16.888 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc
02:16:16.895 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:16:16.896 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc
02:16:16.898 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:16:16.902 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:16:16.905 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:16:16.906 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:16:16.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:16.916669 31823 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:16.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:16.917068 31823 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:16.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:16 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529776, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:16.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:16:16.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:16.944 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc
02:16:16.945 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:16:16.946 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.204.253:37985
02:16:16.948 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:16:16.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.948992 31539 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 31644
02:16:16.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.911228 31721 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: OK, instance_id { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" instance_seqno: 1714529772777595 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:16:16.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.950191 31721 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:16.975 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:16.975 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.204.252:37931
02:16:16.976 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:16:16.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:16.976414 31539 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 31721
02:16:17.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:17.000 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.204.254:33425
02:16:17.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:16:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:17.001247 31539 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 31567
02:16:17.026 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:17.026 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.204.253:37985
02:16:17.028 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:17.030952 31539 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:17.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/logs
02:16:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/info.pb
02:16:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.253:37985
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.253
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35677
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.253.keytab
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.253
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:16:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.367558 31827 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.368175 31827 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:17.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.368858 31827 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.410197 31827 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:16:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.410470 31827 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:16:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.410744 31827 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.410959 31827 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:16:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.411166 31827 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:16:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.411401 31827 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:16:17.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.411554 31827 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44721/jwks.json
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:17.451700 31827 master_runner.cc:386] Master server non-default flags:
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.253.keytab
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.253
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.253:37985
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/info.pb
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.253
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35677
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/logs
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:17.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:17.453249 31827 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:17.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:17.454702 31827 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:17.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.475219 31834 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:18.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.478215 31837 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:17.476071 31835 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:18.640377 31836 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:16:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.640444 31827 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:16:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.645362 31827 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:18.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.648861 31827 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:18.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.650425 31827 hybrid_clock.cc:648] HybridClock initialized: now 1714529778650370 us; error 66 us; skew 500 ppm
02:16:18.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.651397 31827 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:18.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:18 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529778, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:18.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.657895 31827 init.cc:377] Logged in from keytab as kudu/127.30.204.253@KRBTEST.COM (short username kudu)
02:16:18.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.664527 31827 webserver.cc:460] Webserver started at http://127.30.204.253:35677/ using document root <none> and password file <none>
02:16:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.665644 31827 fs_manager.cc:359] Metadata directory not provided
02:16:18.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.665879 31827 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:18.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.674433 31827 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
02:16:18.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.679388 31846 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:18.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.680624 31827 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:16:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.681053 31827 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c71067590384d838bbf8f09fec01268"
02:16:18.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:14 on dist-test-slave-tmq0"
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.683601 31827 fs_report.cc:383] FS layout report
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/data
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:18.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.739748 31827 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:18.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.748303 31827 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:18.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.748963 31827 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.826310 31827 rpc_server.cc:307] RPC server started. Bound to: 127.30.204.253:37985
02:16:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.826383 31897 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.204.253:37985 every 8 connection(s)
02:16:18.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.829476 31827 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/data/info.pb
02:16:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.839303 31539 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31827
02:16:18.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:18.840 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.204.252:37931
02:16:18.840 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:16:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.842244 31898 sys_catalog.cc:263] Verifying existing consensus state
02:16:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.842432 31539 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/logs
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/info.pb
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.252:37931
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.252
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46195
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.252.keytab
02:16:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.252
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240501 02:16:18.843283 31898 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:16:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.843940 31827 master.cc:537] Master@127.30.204.253:37985 shutting down...
02:16:18.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:18.911039 31827 master.cc:552] Master@127.30.204.253:37985 shutdown complete.
02:16:18.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:16:19.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.184541 31900 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:19.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.185266 31900 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:19.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.186265 31900 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.220773 31900 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:16:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.221084 31900 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:16:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.221527 31900 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.221874 31900 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:16:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.222222 31900 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:16:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.222555 31900 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:16:19.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.222790 31900 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44721/jwks.json
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:19.262501 31900 master_runner.cc:386] Master server non-default flags:
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.252.keytab
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.252
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.252:37931
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/info.pb
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.252
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46195
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/logs
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:19.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:19.264237 31900 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:19.265718 31900 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:19.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.282258 31906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:20.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.283177 31909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:20.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:19.283512 31907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:20.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:19.286047 31900 server_base.cc:1034] running on GCE node
02:16:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.504496 31900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.507303 31900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:20.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.508769 31900 hybrid_clock.cc:648] HybridClock initialized: now 1714529780508700 us; error 46 us; skew 500 ppm
02:16:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.509829 31900 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:20 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529780, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:20.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.519973 31900 init.cc:377] Logged in from keytab as kudu/127.30.204.252@KRBTEST.COM (short username kudu)
02:16:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.526027 31900 webserver.cc:460] Webserver started at http://127.30.204.252:46195/ using document root <none> and password file <none>
02:16:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.527029 31900 fs_manager.cc:359] Metadata directory not provided
02:16:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.527348 31900 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.535975 31900 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
02:16:20.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.540719 31919 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:20.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.541915 31900 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
02:16:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.542245 31900 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82e719f776d648f5af481807d2efb29b"
02:16:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:16 on dist-test-slave-tmq0"
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.544746 31900 fs_report.cc:383] FS layout report
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/wal
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/data
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.605069 31900 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.610934 31900 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.611485 31900 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.690853 31900 rpc_server.cc:307] RPC server started. Bound to: 127.30.204.252:37931
02:16:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.690991 31970 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.204.252:37931 every 8 connection(s)
02:16:20.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.693929 31900 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-2/data/info.pb
02:16:20.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.694941 31539 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31900
02:16:20.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:20.696 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.204.254:33425
02:16:20.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.698949 31539 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:16:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/logs
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/info.pb
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.254:33425
02:16:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.254
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38703
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.254.keytab
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.254
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:16:20.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.707597 31971 sys_catalog.cc:263] Verifying existing consensus state
02:16:20.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.713382 31971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Bootstrap starting.
02:16:20.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.751442 31971 log.cc:824] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Log is configured to *not* fsync() on all Append() calls
02:16:20.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.758208 31971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:16:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.759157 31971 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Bootstrap complete.
02:16:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.792413 31971 raft_consensus.cc:348] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.795298 31971 raft_consensus.cc:374] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:16:20.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.795631 31971 raft_consensus.cc:729] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 82e719f776d648f5af481807d2efb29b, State: Initialized, Role: FOLLOWER
02:16:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.796581 31971 consensus_queue.cc:260] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.800041 31975 sys_catalog.cc:455] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } } }
02:16:20.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.801084 31975 sys_catalog.cc:458] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [sys.catalog]: This master's current role is: FOLLOWER
02:16:20.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.801399 31971 sys_catalog.cc:564] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [sys.catalog]: configured and running, proceeding with master startup.
02:16:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.853433 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:20.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.853818 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.830050 31900 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111),
02:16:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.864789 31900 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.867666 31925 tablet.cc:2369] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:16:20.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.934441 31975 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:20.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.935038 31975 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:20.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.938629 31975 leader_election.cc:288] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425), 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985)
02:16:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.941625 31921 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:20.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.946872 31921 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425): Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.947801 31923 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985): Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.948279 31923 leader_election.cc:302] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82e719f776d648f5af481807d2efb29b; no voters: 0c5f99e2f8dd4b67a363a5b7e74d26ae, 8c71067590384d838bbf8f09fec01268
02:16:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:20.950121 31975 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:16:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.996279 31383 debug-util.cc:398] Leaking SignalData structure 0x7b08000b08c0 after lost signal to thread 31257
02:16:20.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:20.997603 31383 debug-util.cc:398] Leaking SignalData structure 0x7b08000be820 after lost signal to thread 31386
02:16:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.178639 31973 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:16:21.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.179586 31973 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:16:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.180806 31973 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:16:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.249171 31973 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:16:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.249590 31973 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:16:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.250150 31973 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:16:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.250526 31973 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:16:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.250888 31973 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:16:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.251245 31973 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:16:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.251538 31973 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44721/jwks.json
02:16:21.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:21.330282 31973 master_runner.cc:386] Master server non-default flags:
02:16:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:16:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.204.212:40139
02:16:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:16:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:16:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data
02:16:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.204.254:33425,127.30.204.253:37985,127.30.204.252:37931
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/krb5kdc/kudu_127.30.204.254.keytab
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.204.254
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.204.254:33425
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44721/jwks.json
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:16:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/info.pb
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.204.254
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38703
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/logs
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 01:43:23 UTC on 1a15035a6830
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 68
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:16:21.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:21.332201 31973 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:16:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:21.333814 31973 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:16:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.355091 31996 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.859656 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.860082 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:22.568341 32002 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:22.569034 32002 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:22.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:22.583434 32002 leader_election.cc:288] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425), 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985)
02:16:22.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:22.603364 31921 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425): Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:22.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:22.604816 31923 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985): Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:22.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:22.605356 31923 leader_election.cc:302] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82e719f776d648f5af481807d2efb29b; no voters: 0c5f99e2f8dd4b67a363a5b7e74d26ae, 8c71067590384d838bbf8f09fec01268
02:16:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:22.606294 32002 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:16:22.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:22.865168 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:22.865999 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:22.889549 31524 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0a80 after lost signal to thread 31398
02:16:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:22.890976 31524 debug-util.cc:398] Leaking SignalData structure 0x7b08000b06c0 after lost signal to thread 31527
02:16:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:23.871701 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:23.872716 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:22.751652 31995 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 31973
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:24.854768 31995 kernel_stack_watchdog.cc:198] Thread 31973 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 392ms:
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:21.357021 31997 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:24.857748 31999 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:16:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:24.858412 31998 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:16:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:24.859364 31973 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.502s user 1.126s sys 1.570s
02:16:24.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:24.860034 31973 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.503s user 1.127s sys 1.570s
02:16:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.860852 31973 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:16:24.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.862785 31973 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:16:24.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.873153 31973 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:16:24.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.877295 31973 hybrid_clock.cc:648] HybridClock initialized: now 1714529784877190 us; error 118 us; skew 500 ppm
02:16:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:24.878438 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:24.878993 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.879465 31973 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:16:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:24 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529784, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.890568 31973 init.cc:377] Logged in from keytab as kudu/127.30.204.254@KRBTEST.COM (short username kudu)
02:16:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.897758 31973 webserver.cc:460] Webserver started at http://127.30.204.254:38703/ using document root <none> and password file <none>
02:16:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.899478 31973 fs_manager.cc:359] Metadata directory not provided
02:16:24.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.899825 31973 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:16:24.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.913828 31973 fs_manager.cc:709] Time spent opening directory manager: real 0.008s user 0.005s sys 0.003s
02:16:24.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.921283 32012 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:16:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.922983 31973 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
02:16:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.923525 31973 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data,/tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae"
02:16:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 02:16:12 on dist-test-slave-tmq0"
02:16:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.927692 31973 fs_report.cc:383] FS layout report
02:16:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:16:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal
02:16:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/data
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:16:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:16:24.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.983026 32013 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:24.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:24.984225 32013 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:25.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.000931 32013 leader_election.cc:288] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425), 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985)
02:16:25.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:25.027963 31923 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985): Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:25.029036 31921 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425): Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.029335 31973 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:16:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.029704 31921 leader_election.cc:302] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82e719f776d648f5af481807d2efb29b; no voters: 0c5f99e2f8dd4b67a363a5b7e74d26ae, 8c71067590384d838bbf8f09fec01268
02:16:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.030635 32013 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:16:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.038726 31973 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:16:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.039448 31973 kserver.cc:163] Server-wide thread pool size limit: 3276
02:16:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.136508 32067 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.204.254:33425 every 8 connection(s)
02:16:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.136492 31973 rpc_server.cc:307] RPC server started. Bound to: 127.30.204.254:33425
02:16:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.141373 31973 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/data/info.pb
02:16:25.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.144834 31539 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu as pid 31973
02:16:25.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:16:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.157915 32068 sys_catalog.cc:263] Verifying existing consensus state
02:16:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240501 02:16:25.159224 32068 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:16:25.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.170086 31973 master.cc:537] Master@127.30.204.254:33425 shutting down...
02:16:25.230 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:25.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.305790 31973 master.cc:552] Master@127.30.204.254:33425 shutdown complete.
02:16:25.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskjrmL5D/test-tmp/mini-kudu-cluster3771774913793265360/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:16:25.325 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.204.252:37931(127.30.204.252:37931)] connecting to peer
02:16:25.335 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:25.415 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.204.252:37931(127.30.204.252:37931)] Successfully connected to peer
02:16:25.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.419 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.428 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.529 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:16:25.540 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.204.252
02:16:25.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:25 dist-test-slave-tmq0 krb5kdc[31551](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529776, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.204.252@KRBTEST.COM
02:16:25.816 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf4ebdc9b, L:/127.0.0.1:47232 - R:/127.30.204.252:37931] using SASL/GSSAPI
02:16:25.842 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:25.854 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:25.859 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:25.864 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.864 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.865 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.868563 31900 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111),
02:16:25.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:25.869076 31900 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:25.870 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.870 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.871 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.873 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:25.883754 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:25.884130 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:25.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:25.896 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:25.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.899 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.900 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.904 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.904 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.905 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:25.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:25.915 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:25.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.920 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.922 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.923 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.925 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:25.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:25.935 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:25.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.941 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.944 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.946 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:25.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:25.975 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:25.978 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:25.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.981 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:25.983 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:26.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:26.016 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:26.021 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.021 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.021 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.030 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.030 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.037 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:26.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:26.096 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:26.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.103 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.107 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.110 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:26.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:26.316 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:26.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.318 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.321 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.322 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.323 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.323 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.324 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:26.453 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:26.455 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:26.457 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.457 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.458 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.462 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:26.558817 32078 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:26.559365 32078 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:26.561559 32078 leader_election.cc:288] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425), 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985)
02:16:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:26.563875 31921 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111) [suppressed 5 similar messages]
02:16:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:26.567587 31921 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425): Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:26.568089 31923 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985): Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:26.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:26.568400 31923 leader_election.cc:302] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82e719f776d648f5af481807d2efb29b; no voters: 0c5f99e2f8dd4b67a363a5b7e74d26ae, 8c71067590384d838bbf8f09fec01268
02:16:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:26.569033 32078 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:16:26.694 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:26.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:26.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:26.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:26.700 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:26.887692 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:26.888068 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:27.653 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:27.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:27.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:27.657 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:27.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:27.657 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:27.657 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:27.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:27.659 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:27.891137 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:27.891474 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:28.895265 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:28.895686 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:29.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:29.577699 32086 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:29.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:29.578634 32086 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:29.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:29.581961 32086 leader_election.cc:288] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425), 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985)
02:16:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:29.588773 31921 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425): Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:29.589366 31923 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985): Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:29.589725 31923 leader_election.cc:302] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82e719f776d648f5af481807d2efb29b; no voters: 0c5f99e2f8dd4b67a363a5b7e74d26ae, 8c71067590384d838bbf8f09fec01268
02:16:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:29.590515 32086 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:16:29.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:29.899891 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:29.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:29.900523 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:30.473 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:30.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:30.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:30.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:30.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:30.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:30.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:30.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:30.482 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:30 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529790, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:30.524214 31915 init.cc:295] Successfully reacquired a new kerberos TGT
02:16:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:30.872511 31900 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111),
02:16:30.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:30.873016 31900 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:30.904302 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:30.904605 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:31.907677 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:31.908002 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:32.910800 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:32.911125 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:33.914089 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:33.914389 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:34.414 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:34.415 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:34.417 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:34.417 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:34.417 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:34.417 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:34.417 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:34.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:34.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:34.917311 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:34.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:34.917646 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:35.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:35.795 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:35.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:35.797 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:35.797 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:35.797 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:35.798 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:35.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:35.799 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:35.875798 31900 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111),
02:16:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:35.876156 31900 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:35.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:35.921267 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:35.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:35.921569 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:36.093 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:36.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:36.097 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:36.097 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:36.097 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:36.097 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:36.097 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:36.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:36.099 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:36.705572 32098 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:36.705986 32098 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:36.707680 32098 leader_election.cc:288] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425), 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985)
02:16:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:36.710038 31921 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111) [suppressed 3 similar messages]
02:16:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:36.712884 31923 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985): Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:36.713376 31921 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425): Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:36.713676 31921 leader_election.cc:302] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82e719f776d648f5af481807d2efb29b; no voters: 0c5f99e2f8dd4b67a363a5b7e74d26ae, 8c71067590384d838bbf8f09fec01268
02:16:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:36.714206 32098 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:16:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:36.924842 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:36.925119 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:37.928083 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:37.928398 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:38.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:38.931655 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:38.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:38.931957 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:39.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:39.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:39.717 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:39.717 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:39.717 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:39.717 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:39.718 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:39.718 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:39.720 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:39.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:39.935644 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:39.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:39.936048 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:40.474 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:40.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:40.477 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:40.477 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:40.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:40.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:40.479 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:40.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:40.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:40.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:40 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529800, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:40.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:40.526863 31915 init.cc:295] Successfully reacquired a new kerberos TGT
02:16:40.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:40.880342 31900 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111),
02:16:40.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:40.880826 31900 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:40.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:40.939122 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:40.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:40.939518 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:41.433 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:41.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:41.437 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:41.437 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:41.437 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:41.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:41.437 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:41.437 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:41.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:41.833 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:41.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:41.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:41.837 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:41.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:41.840 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:41.840 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:41.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:41.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:41.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:41.942473 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:41.942796 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:42.173 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:42.174 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:42.175 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:42.175 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:42.175 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:42.175 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:42.175 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:42.175 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:42.178 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:42.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:42.945787 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:42.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:42.946110 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:43.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:43.948797 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:43.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:43.949092 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:44.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:44.714 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:44.715 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:44.715 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:44.715 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:44.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:44.716 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:44.716 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:44.718 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:44.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:44.952144 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:44.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:44.952471 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:45.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:45.592968 30965 debug-util.cc:398] Leaking SignalData structure 0x7b08000545e0 after lost signal to thread 30900
02:16:45.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:45.593753 30965 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1460 after lost signal to thread 30968
02:16:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:45.884156 31900 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111),
02:16:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:45.884713 31900 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:45.955617 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:45.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:45.955942 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:46.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:46.674 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:46.675 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:46.675 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:46.675 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:46.676 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:46.675 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:46.676 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:46.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:46.960809 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:46.961266 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:47.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:47.964283 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:47.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:47.964588 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:48.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:48.967628 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:48.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:48.967913 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:49.971621 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:49.971982 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:50.113 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:50.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:50.115 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.115 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.115 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.115 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.119 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:50.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 01 02:16:50 dist-test-slave-tmq0 krb5kdc[31551](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714529810, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.204.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:16:50.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:50.529489 31915 init.cc:295] Successfully reacquired a new kerberos TGT
02:16:50.533 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:50.534 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:50.535 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.535 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.535 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.535 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.535 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.535 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.538 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:50.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:50.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:50.856 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.856 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.857 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.857 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.857 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:50.857 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:50.859 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:50.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:50.889436 31900 master_runner.cc:186] Error getting master registration for 127.30.204.254:33425: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111),
02:16:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:50.889914 31900 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:16:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:50.974804 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:50.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:50.975103 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:51.180075 32120 raft_consensus.cc:482] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:16:51.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:51.180636 32120 raft_consensus.cc:504] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c5f99e2f8dd4b67a363a5b7e74d26ae" member_type: VOTER last_known_addr { host: "127.30.204.254" port: 33425 } } peers { permanent_uuid: "8c71067590384d838bbf8f09fec01268" member_type: VOTER last_known_addr { host: "127.30.204.253" port: 37985 } } peers { permanent_uuid: "82e719f776d648f5af481807d2efb29b" member_type: VOTER last_known_addr { host: "127.30.204.252" port: 37931 } }
02:16:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:51.182770 32120 leader_election.cc:288] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425), 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985)
02:16:51.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:51.184254 31921 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:16:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:51.187990 31921 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0c5f99e2f8dd4b67a363a5b7e74d26ae (127.30.204.254:33425): Network error: Client connection negotiation failed: client connection to 127.30.204.254:33425: connect: Connection refused (error 111)
02:16:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:51.188642 31923 leader_election.cc:334] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8c71067590384d838bbf8f09fec01268 (127.30.204.253:37985): Network error: Client connection negotiation failed: client connection to 127.30.204.253:37985: connect: Connection refused (error 111)
02:16:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:51.188977 31923 leader_election.cc:302] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 82e719f776d648f5af481807d2efb29b; no voters: 0c5f99e2f8dd4b67a363a5b7e74d26ae, 8c71067590384d838bbf8f09fec01268
02:16:51.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:51.189903 32120 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:16:51.773 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.253:37985(127.30.204.253:37985)] connecting to peer
02:16:51.774 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.204.254:33425(127.30.204.254:33425)] connecting to peer
02:16:51.775 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:51.775 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.204.254:33425(127.30.204.254:33425)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:51.775 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:51.775 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.253:37985(127.30.204.253:37985)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:51.776 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.253:37985: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985
02:16:51.775 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.204.254:33425: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425
02:16:51.778 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425; will retry
02:16:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:51.977908 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:51.978206 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:52.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:52.980962 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:52.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:52.981248 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:53.769011 31383 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0140 after lost signal to thread 31257
02:16:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:53.769845 31383 debug-util.cc:398] Leaking SignalData structure 0x7b08000be940 after lost signal to thread 31386
02:16:53.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:53.983965 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:53.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:53.984248 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:54.987201 31986 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 82e719f776d648f5af481807d2efb29b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:16:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 02:16:54.987577 31986 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:16:55.157 [DEBUG - pool-6-thread-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=30001), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [263ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [273ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [679ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [707ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [713ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [715ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [716ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [742ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [745ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [748ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [762ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [765ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [766ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [768ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [789ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [790ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [822ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [825ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [864ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [872ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [873ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [881ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [946ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [947ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [951ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [953ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1156ms] refreshing cache from master, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1165ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1167ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1305ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1543ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1544ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [2496ms] refreshing cache from master, [2496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [2498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [2500ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [2500ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [2502ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [5316ms] refreshing cache from master, [5316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [5317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [5317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [5322ms] too many traces: truncated at 100 traces, [5322ms] too many traces: truncated at 100 traces, deferred=Deferred@830972006(state=PENDING, result=null, callback=(continuation of Deferred@2076269870 after retry RPC after error@478280414) -> (continuation of Deferred@1594860073 after retry RPC after error@1401404734) -> (continuation of Deferred@585019981 after retry RPC after error@823847874) -> (continuation of Deferred@321259810 after retry RPC after error@2099656757) -> (continuation of Deferred@717074694 after retry RPC after error@1285491560) -> (continuation of Deferred@1867740352 after retry RPC after error@595074703) -> (continuation of Deferred@1686810215 after retry RPC after error@1358577109) -> (continuation of Deferred@612087422 after retry RPC after error@1192515608) -> (continuation of Deferred@1985752843 after retry RPC after error@2009702528) -> (continuation of Deferred@251146305 after retry RPC after error@1799978270) -> (continuation of Deferred@2096767755 after retry RPC after error@1183666612) -> (continuation of Deferred@154671900 after retry RPC after error@699999912) -> (continuation of Deferred@74885871 after retry RPC after error@677006116) -> (continuation of Deferred@1643333064 after retry RPC after error@18854712) -> (continuation of Deferred@806309926 after retry RPC after error@1249259112) -> (continuation of Deferred@835776491 after retry RPC after error@1294601370) -> (continuation of Deferred@124713849 after retry RPC after error@2081460370) -> (continuation of Deferred@1250527052 after retry RPC after error@1465523291) -> (continuation of Deferred@664764666 after retry RPC after error@1609760923) -> (continuation of Deferred@1497784270 after retry RPC after error@1593387129) -> (continuation of Deferred@87747340 after retry RPC after error@1108002024) -> (continuation of Deferred@702106440 after retry RPC after error@1056179434) -> (continuation of Deferred@1816455859 after retry RPC after error@2029870679) -> (continuation of Deferred@454940939 after retry RPC after error@89152982) -> (continuation of Deferred@1317678781 after retry RPC after error@1545064157) -> (continuation of Deferred@691503573 after retry RPC after error@781113288) -> (continuation of Deferred@829352802 after retry RPC after error@1399696052), errback=(continuation of Deferred@2076269870 after retry RPC after error@478280414) -> (continuation of Deferred@1594860073 after retry RPC after error@1401404734) -> (continuation of Deferred@585019981 after retry RPC after error@823847874) -> (continuation of Deferred@321259810 after retry RPC after error@2099656757) -> (continuation of Deferred@717074694 after retry RPC after error@1285491560) -> (continuation of Deferred@1867740352 after retry RPC after error@595074703) -> (continuation of Deferred@1686810215 after retry RPC after error@1358577109) -> (continuation of Deferred@612087422 after retry RPC after error@1192515608) -> (continuation of Deferred@1985752843 after retry RPC after error@2009702528) -> (continuation of Deferred@251146305 after retry RPC after error@1799978270) -> (continuation of Deferred@2096767755 after retry RPC after error@1183666612) -> (continuation of Deferred@154671900 after retry RPC after error@699999912) -> (continuation of Deferred@74885871 after retry RPC after error@677006116) -> (continuation of Deferred@1643333064 after retry RPC after error@18854712) -> (continuation of Deferred@806309926 after retry RPC after error@1249259112) -> (continuation of Deferred@835776491 after retry RPC after error@1294601370) -> (continuation of Deferred@124713849 after retry RPC after error@2081460370) -> (continuation of Deferred@1250527052 after retry RPC after error@1465523291) -> (continuation of Deferred@664764666 after retry RPC after error@1609760923) -> (continuation of Deferred@1497784270 after retry RPC after error@1593387129) -> (continuation of Deferred@87747340 after retry RPC after error@1108002024) -> (continuation of Deferred@702106440 after retry RPC after error@1056179434) -> (continuation of Deferred@1816455859 after retry RPC after error@2029870679) -> (continuation of Deferred@454940939 after retry RPC after error@89152982) -> (continuation of Deferred@1317678781 after retry RPC after error@1545064157) -> (continuation of Deferred@691503573 after retry RPC after error@781113288) -> (continuation of Deferred@829352802 after retry RPC after error@1399696052)))
02:16:55.161 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.204.254:33425(127.30.204.254:33425)] disconnecting while in state TERMINATED
02:16:55.162 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.204.253:37985(127.30.204.253:37985)] disconnecting while in state TERMINATED
02:16:55.163 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.204.252:37931(127.30.204.252:37931)] disconnecting while in state READY
02:16:55.169 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:16:55.171 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.204.252:37931(127.30.204.252:37931)] handling channelInactive
02:16:55.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.204.252:37931(127.30.204.252:37931)] cleaning up while in state READY due to: connection closed
02:16:55.172 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:16:55.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:55.184190 31539 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 31900
02:16:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:16:56Z chronyd exiting
02:16:57.359 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:16:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:57.360508 30814 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 31109
02:16:57.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:57.388545 30814 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 31254
02:16:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:57.417965 30814 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 31395
02:16:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:57.448638 30814 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 30826
02:16:57.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:57.484866 30814 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 30896
02:16:57.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 02:16:57.520561 30814 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskjrmL5D/build/tsan/bin/kudu with pid 30971
02:16:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T02:16:58Z chronyd exiting
02:16:59.635 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=30001), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [263ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [273ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [679ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [707ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [713ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [715ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [716ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [742ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [745ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [748ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [762ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [765ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [766ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [768ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [789ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [790ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [822ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [825ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [864ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [872ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [873ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [881ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [946ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [947ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [951ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [953ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1156ms] refreshing cache from master, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1165ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1167ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1305ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1543ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1544ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [2496ms] refreshing cache from master, [2496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [2498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [2500ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [2500ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [2502ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [5316ms] refreshing cache from master, [5316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [5317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [5317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [5322ms] too many traces: truncated at 100 traces, [5322ms] too many traces: truncated at 100 traces, deferred=Deferred@830972006(state=PENDING, result=null, callback=(continuation of Deferred@2076269870 after retry RPC after error@478280414) -> (continuation of Deferred@1594860073 after retry RPC after error@1401404734) -> (continuation of Deferred@585019981 after retry RPC after error@823847874) -> (continuation of Deferred@321259810 after retry RPC after error@2099656757) -> (continuation of Deferred@717074694 after retry RPC after error@1285491560) -> (continuation of Deferred@1867740352 after retry RPC after error@595074703) -> (continuation of Deferred@1686810215 after retry RPC after error@1358577109) -> (continuation of Deferred@612087422 after retry RPC after error@1192515608) -> (continuation of Deferred@1985752843 after retry RPC after error@2009702528) -> (continuation of Deferred@251146305 after retry RPC after error@1799978270) -> (continuation of Deferred@2096767755 after retry RPC after error@1183666612) -> (continuation of Deferred@154671900 after retry RPC after error@699999912) -> (continuation of Deferred@74885871 after retry RPC after error@677006116) -> (continuation of Deferred@1643333064 after retry RPC after error@18854712) -> (continuation of Deferred@806309926 after retry RPC after error@1249259112) -> (continuation of Deferred@835776491 after retry RPC after error@1294601370) -> (continuation of Deferred@124713849 after retry RPC after error@2081460370) -> (continuation of Deferred@1250527052 after retry RPC after error@1465523291) -> (continuation of Deferred@664764666 after retry RPC after error@1609760923) -> (continuation of Deferred@1497784270 after retry RPC after error@1593387129) -> (continuation of Deferred@87747340 after retry RPC after error@1108002024) -> (continuation of Deferred@702106440 after retry RPC after error@1056179434) -> (continuation of Deferred@1816455859 after retry RPC after error@2029870679) -> (continuation of Deferred@454940939 after retry RPC after error@89152982) -> (continuation of Deferred@1317678781 after retry RPC after error@1545064157) -> (continuation of Deferred@691503573 after retry RPC after error@781113288) -> (continuation of Deferred@829352802 after retry RPC after error@1399696052), errback=(continuation of Deferred@2076269870 after retry RPC after error@478280414) -> (continuation of Deferred@1594860073 after retry RPC after error@1401404734) -> (continuation of Deferred@585019981 after retry RPC after error@823847874) -> (continuation of Deferred@321259810 after retry RPC after error@2099656757) -> (continuation of Deferred@717074694 after retry RPC after error@1285491560) -> (continuation of Deferred@1867740352 after retry RPC after error@595074703) -> (continuation of Deferred@1686810215 after retry RPC after error@1358577109) -> (continuation of Deferred@612087422 after retry RPC after error@1192515608) -> (continuation of Deferred@1985752843 after retry RPC after error@2009702528) -> (continuation of Deferred@251146305 after retry RPC after error@1799978270) -> (continuation of Deferred@2096767755 after retry RPC after error@1183666612) -> (continuation of Deferred@154671900 after retry RPC after error@699999912) -> (continuation of Deferred@74885871 after retry RPC after error@677006116) -> (continuation of Deferred@1643333064 after retry RPC after error@18854712) -> (continuation of Deferred@806309926 after retry RPC after error@1249259112) -> (continuation of Deferred@835776491 after retry RPC after error@1294601370) -> (continuation of Deferred@124713849 after retry RPC after error@2081460370) -> (continuation of Deferred@1250527052 after retry RPC after error@1465523291) -> (continuation of Deferred@664764666 after retry RPC after error@1609760923) -> (continuation of Deferred@1497784270 after retry RPC after error@1593387129) -> (continuation of Deferred@87747340 after retry RPC after error@1108002024) -> (continuation of Deferred@702106440 after retry RPC after error@1056179434) -> (continuation of Deferred@1816455859 after retry RPC after error@2029870679) -> (continuation of Deferred@454940939 after retry RPC after error@89152982) -> (continuation of Deferred@1317678781 after retry RPC after error@1545064157) -> (continuation of Deferred@691503573 after retry RPC after error@781113288) -> (continuation of Deferred@829352802 after retry RPC after error@1399696052)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1407) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=30001), Traces: [0ms] refreshing cache from master, [52ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [263ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [273ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [679ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [686ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [696ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [707ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [713ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [715ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [716ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [742ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [745ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [747ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [748ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [762ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [765ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [766ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [768ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [789ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [790ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [822ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [825ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [864ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [872ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [873ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [881ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [935ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [946ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [947ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [951ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [953ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1156ms] refreshing cache from master, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1165ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1166ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1167ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1295ms] refreshing cache from master, [1296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1301ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1305ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1536ms] refreshing cache from master, [1537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [1539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [1539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [1543ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [1544ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [2496ms] refreshing cache from master, [2496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [2497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [2498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [2500ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.254:33425: Network error: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [2500ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.253:37985: Network error: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985, [2502ms] Sub RPC ConnectToMaster: received response from server master-127.30.204.252:37931: OK, [2502ms] delaying RPC due to: Service unavailable: Master config (127.30.204.253:37985,127.30.204.252:37931,127.30.204.254:33425) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.253:37985(127.30.204.253:37985): Connection refused: /127.30.204.253:37985,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.204.254:33425(127.30.204.254:33425): Connection refused: /127.30.204.254:33425, [5316ms] refreshing cache from master, [5316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.253:37985, [5317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.252:37931, [5317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.204.254:33425, [5322ms] too many traces: truncated at 100 traces, [5322ms] too many traces: truncated at 100 traces, deferred=Deferred@830972006(state=PENDING, result=null, callback=(continuation of Deferred@2076269870 after retry RPC after error@478280414) -> (continuation of Deferred@1594860073 after retry RPC after error@1401404734) -> (continuation of Deferred@585019981 after retry RPC after error@823847874) -> (continuation of Deferred@321259810 after retry RPC after error@2099656757) -> (continuation of Deferred@717074694 after retry RPC after error@1285491560) -> (continuation of Deferred@1867740352 after retry RPC after error@595074703) -> (continuation of Deferred@1686810215 after retry RPC after error@1358577109) -> (continuation of Deferred@612087422 after retry RPC after error@1192515608) -> (continuation of Deferred@1985752843 after retry RPC after error@2009702528) -> (continuation of Deferred@251146305 after retry RPC after error@1799978270) -> (continuation of Deferred@2096767755 after retry RPC after error@1183666612) -> (continuation of Deferred@154671900 after retry RPC after error@699999912) -> (continuation of Deferred@74885871 after retry RPC after error@677006116) -> (continuation of Deferred@1643333064 after retry RPC after error@18854712) -> (continuation of Deferred@806309926 after retry RPC after error@1249259112) -> (continuation of Deferred@835776491 after retry RPC after error@1294601370) -> (continuation of Deferred@124713849 after retry RPC after error@2081460370) -> (continuation of Deferred@1250527052 after retry RPC after error@1465523291) -> (continuation of Deferred@664764666 after retry RPC after error@1609760923) -> (continuation of Deferred@1497784270 after retry RPC after error@1593387129) -> (continuation of Deferred@87747340 after retry RPC after error@1108002024) -> (continuation of Deferred@702106440 after retry RPC after error@1056179434) -> (continuation of Deferred@1816455859 after retry RPC after error@2029870679) -> (continuation of Deferred@454940939 after retry RPC after error@89152982) -> (continuation of Deferred@1317678781 after retry RPC after error@1545064157) -> (continuation of Deferred@691503573 after retry RPC after error@781113288) -> (continuation of Deferred@829352802 after retry RPC after error@1399696052), errback=(continuation of Deferred@2076269870 after retry RPC after error@478280414) -> (continuation of Deferred@1594860073 after retry RPC after error@1401404734) -> (continuation of Deferred@585019981 after retry RPC after error@823847874) -> (continuation of Deferred@321259810 after retry RPC after error@2099656757) -> (continuation of Deferred@717074694 after retry RPC after error@1285491560) -> (continuation of Deferred@1867740352 after retry RPC after error@595074703) -> (continuation of Deferred@1686810215 after retry RPC after error@1358577109) -> (continuation of Deferred@612087422 after retry RPC after error@1192515608) -> (continuation of Deferred@1985752843 after retry RPC after error@2009702528) -> (continuation of Deferred@251146305 after retry RPC after error@1799978270) -> (continuation of Deferred@2096767755 after retry RPC after error@1183666612) -> (continuation of Deferred@154671900 after retry RPC after error@699999912) -> (continuation of Deferred@74885871 after retry RPC after error@677006116) -> (continuation of Deferred@1643333064 after retry RPC after error@18854712) -> (continuation of Deferred@806309926 after retry RPC after error@1249259112) -> (continuation of Deferred@835776491 after retry RPC after error@1294601370) -> (continuation of Deferred@124713849 after retry RPC after error@2081460370) -> (continuation of Deferred@1250527052 after retry RPC after error@1465523291) -> (continuation of Deferred@664764666 after retry RPC after error@1609760923) -> (continuation of Deferred@1497784270 after retry RPC after error@1593387129) -> (continuation of Deferred@87747340 after retry RPC after error@1108002024) -> (continuation of Deferred@702106440 after retry RPC after error@1056179434) -> (continuation of Deferred@1816455859 after retry RPC after error@2029870679) -> (continuation of Deferred@454940939 after retry RPC after error@89152982) -> (continuation of Deferred@1317678781 after retry RPC after error@1545064157) -> (continuation of Deferred@691503573 after retry RPC after error@781113288) -> (continuation of Deferred@829352802 after retry RPC after error@1399696052)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1407)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
... 1 more