17:10:46.308 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
17:10:46.320 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNB5BOS/test-tmp
17:10:46.323 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
17:10:46.326 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNB5BOS/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
17:10:46.538 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
17:10:46.721 [DEBUG - main] (MiniKuduCluster.java:195) Response:
17:10:46.722 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
17:10:46.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:46.726769 15055 mini_oidc.cc:152] Starting JWKS server
17:10:46.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:46.730175 15055 webserver.cc:460] Webserver started at http://127.0.0.1:41921/ using document root <none> and password file <none>
17:10:46.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:46.730620 15055 mini_oidc.cc:189] Starting OIDC Discovery server
17:10:46.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:46.732760 15055 webserver.cc:460] Webserver started at http://127.0.0.1:44241/ using document root <none> and password file <none>
17:10:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T17:10:46Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
17:10:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T17:10:46Z Disabled control of system clock
17:10:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:46.798116 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/wal
17:10:46.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data
17:10:46.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:10:46.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:10:46.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:10:46.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/logs
17:10:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data/info.pb
17:10:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:10:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:10:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:10:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.254:39071
17:10:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.254
17:10:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:10:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:10:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41921/jwks.json with env {}
17:10:47.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.090252 15067 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:10:47.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.090801 15067 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:10:47.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.091533 15067 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:10:47.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.121451 15067 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:10:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.121731 15067 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:10:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.121922 15067 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:10:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.122174 15067 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:10:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.122479 15067 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:10:47.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.122715 15067 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41921/jwks.json
17:10:47.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:47.157420 15067 master_runner.cc:386] Master server non-default flags:
17:10:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:47.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data
17:10:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/wal
17:10:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:10:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.254:39071
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41921/jwks.json
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data/info.pb
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.254
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:10:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/logs
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:10:47.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:10:47.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:10:47.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:47.158713 15067 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:10:47.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:47.160810 15067 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:10:47.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.178731 15073 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.179961 15076 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:47.179373 15074 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:48.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:47.180752 15067 server_base.cc:1034] running on GCE node
17:10:48.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.278303 15067 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:10:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.281585 15067 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:10:48.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.283018 15067 hybrid_clock.cc:648] HybridClock initialized: now 1714497048282980 us; error 60 us; skew 500 ppm
17:10:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.283841 15067 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:10:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.302686 15067 webserver.cc:460] Webserver started at http://127.14.179.254:35071/ using document root <none> and password file <none>
17:10:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.303673 15067 fs_manager.cc:359] Metadata directory not provided
17:10:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.303869 15067 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:10:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.304253 15067 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:10:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.309152 15067 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data/instance:
17:10:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7ac01b86aa543249d04624e19665c02"
17:10:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:48 on dist-test-slave-2p34"
17:10:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.310271 15067 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/wal/instance:
17:10:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7ac01b86aa543249d04624e19665c02"
17:10:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:48 on dist-test-slave-2p34"
17:10:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.317479 15067 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
17:10:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.323240 15083 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:10:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.324532 15067 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
17:10:48.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.324857 15067 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/wal
17:10:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7ac01b86aa543249d04624e19665c02"
17:10:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:48 on dist-test-slave-2p34"
17:10:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.325170 15067 fs_report.cc:383] FS layout report
17:10:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:10:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/wal
17:10:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/wal
17:10:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data/data
17:10:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:10:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:10:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:10:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:10:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:10:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:10:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:10:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:10:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:10:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:10:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:10:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.386646 15067 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:10:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.388142 15067 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:10:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.388551 15067 kserver.cc:163] Server-wide thread pool size limit: 3276
17:10:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.455168 15067 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.254:39071
17:10:48.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.455258 15134 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.254:39071 every 8 connection(s)
17:10:48.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.457840 15067 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/data/info.pb
17:10:48.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.462431 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15067
17:10:48.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.462958 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-0/wal/instance
17:10:48.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.463744 15135 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
17:10:48.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.468266 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:48.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:48.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/wal
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/logs
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data/info.pb
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.253:36755
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.253
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:10:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:10:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41921/jwks.json with env {}
17:10:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.478601 15135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } has no permanent_uuid. Determining permanent_uuid...
17:10:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.508281 15135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } has no permanent_uuid. Determining permanent_uuid...
17:10:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.510769 15084 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:48.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.514864 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:36755: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:48.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.564450 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } attempt: 1
17:10:48.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.568765 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:36755: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.637288 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } attempt: 2
17:10:48.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.641494 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:36755: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:48.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.732951 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } attempt: 3
17:10:48.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.737015 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:36755: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.789180 15137 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:10:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.789896 15137 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:10:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.790706 15137 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:10:48.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.823308 15137 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:10:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.823689 15137 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:10:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.824008 15137 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:10:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.824349 15137 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:10:48.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.824741 15137 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:10:48.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.824963 15137 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41921/jwks.json
17:10:48.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.860332 15137 master_runner.cc:386] Master server non-default flags:
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/wal
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.253:36755
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41921/jwks.json
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data/info.pb
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.253
17:10:48.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/logs
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:10:48.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.861879 15137 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:10:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.864141 15137 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:10:48.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.880437 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } attempt: 4
17:10:48.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.881590 15146 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:48.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.885613 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:36755: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:49.184993 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } attempt: 5
17:10:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:49.188964 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:36755: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:49.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:49.736629 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } attempt: 6
17:10:49.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:49.748644 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.253:36755: Network error: Client connection negotiation failed: client connection to 127.14.179.253:36755: connect: Connection refused (error 111)
17:10:49.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.882169 15147 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:49.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:48.884016 15149 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:48.883847 15137 server_base.cc:1034] running on GCE node
17:10:49.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:49.994664 15137 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:10:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:49.997421 15137 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:10:49.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:49.998767 15137 hybrid_clock.cc:648] HybridClock initialized: now 1714497049998713 us; error 68 us; skew 500 ppm
17:10:49.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:49.999756 15137 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:10:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.006127 15137 webserver.cc:460] Webserver started at http://127.14.179.253:39023/ using document root <none> and password file <none>
17:10:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.007019 15137 fs_manager.cc:359] Metadata directory not provided
17:10:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.007202 15137 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:10:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.007653 15137 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:10:50.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.012427 15137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data/instance:
17:10:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e728f9b947d442b09957c7ce5a0514ee"
17:10:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:50 on dist-test-slave-2p34"
17:10:50.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.013492 15137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/wal/instance:
17:10:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e728f9b947d442b09957c7ce5a0514ee"
17:10:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:50 on dist-test-slave-2p34"
17:10:50.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.021355 15137 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
17:10:50.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.026705 15158 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:10:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.027757 15137 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
17:10:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.028095 15137 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/wal
17:10:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e728f9b947d442b09957c7ce5a0514ee"
17:10:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:50 on dist-test-slave-2p34"
17:10:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.028458 15137 fs_report.cc:383] FS layout report
17:10:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:10:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/wal
17:10:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/wal
17:10:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data/data
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:10:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:10:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:10:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.075821 15137 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:10:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.077366 15137 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:10:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.077859 15137 kserver.cc:163] Server-wide thread pool size limit: 3276
17:10:50.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.144201 15137 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.253:36755
17:10:50.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.144277 15209 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.253:36755 every 8 connection(s)
17:10:50.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.146957 15137 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/data/info.pb
17:10:50.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.151844 15210 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
17:10:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.154256 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15137
17:10:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.154604 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-1/wal/instance
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.156874 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/wal
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:50.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/logs
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data/info.pb
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:10:50.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.252:34823
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.252
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:10:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41921/jwks.json with env {}
17:10:50.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.164249 15210 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } has no permanent_uuid. Determining permanent_uuid...
17:10:50.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.188870 15210 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } has no permanent_uuid. Determining permanent_uuid...
17:10:50.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.202839 15210 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } has no permanent_uuid. Determining permanent_uuid...
17:10:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.205417 15162 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.209748 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.259212 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 1
17:10:50.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.263696 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.332162 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 2
17:10:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.336055 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.427428 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 3
17:10:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.431319 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.467427 15212 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:10:50.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.468086 15212 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:10:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.468837 15212 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:10:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.501071 15212 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:10:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.501432 15212 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:10:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.501754 15212 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:10:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.502089 15212 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:10:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.502398 15212 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:10:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.502559 15212 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41921/jwks.json
17:10:50.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.537320 15212 master_runner.cc:386] Master server non-default flags:
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/wal
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.252:34823
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41921/jwks.json
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:50.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data/info.pb
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.252
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/logs
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:10:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:10:50.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:10:50.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:10:50.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:10:50.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.538578 15212 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:10:50.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.540439 15212 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:10:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.557997 15225 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.574777 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 4
17:10:50.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.579308 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.809017 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } attempt: 7
17:10:50.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.835975 15135 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } has no permanent_uuid. Determining permanent_uuid...
17:10:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.841352 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.878729 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 5
17:10:50.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.883313 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.899986 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 1
17:10:50.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.905407 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:50.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.986896 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 2
17:10:50.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.991590 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:51.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.076961 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 3
17:10:51.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:51.081522 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:51.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.221983 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 4
17:10:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:51.226649 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.430706 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 6
17:10:51.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:51.438534 15210 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:51.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.510010 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 5
17:10:51.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:51.513728 15135 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.14.179.252:34823: Network error: Client connection negotiation failed: client connection to 127.14.179.252:34823: connect: Connection refused (error 111)
17:10:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.558485 15223 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:50.559973 15212 server_base.cc:1034] running on GCE node
17:10:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:50.558843 15222 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:51.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.691076 15212 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:10:51.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.694332 15212 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:10:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.695847 15212 hybrid_clock.cc:648] HybridClock initialized: now 1714497051695800 us; error 67 us; skew 500 ppm
17:10:51.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.696645 15212 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:10:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.703428 15212 webserver.cc:460] Webserver started at http://127.14.179.252:38253/ using document root <none> and password file <none>
17:10:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.704329 15212 fs_manager.cc:359] Metadata directory not provided
17:10:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.704535 15212 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:10:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.704950 15212 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:10:51.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.712234 15212 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data/instance:
17:10:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c3bad570dc04c148b8bfd262867b8ff"
17:10:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:51 on dist-test-slave-2p34"
17:10:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.713835 15212 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/wal/instance:
17:10:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c3bad570dc04c148b8bfd262867b8ff"
17:10:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:51 on dist-test-slave-2p34"
17:10:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.723112 15212 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
17:10:51.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.728940 15237 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:10:51.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.730247 15212 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
17:10:51.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.730563 15212 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/wal
17:10:51.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3c3bad570dc04c148b8bfd262867b8ff"
17:10:51.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:51 on dist-test-slave-2p34"
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.730881 15212 fs_report.cc:383] FS layout report
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/wal
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/wal
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data/data
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:10:51.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:10:51.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.803833 15212 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:10:51.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.805253 15212 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:10:51.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.805656 15212 kserver.cc:163] Server-wide thread pool size limit: 3276
17:10:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.883225 15212 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.252:34823
17:10:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.883301 15288 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.252:34823 every 8 connection(s)
17:10:51.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.885778 15212 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/data/info.pb
17:10:51.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.891672 15289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
17:10:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.892374 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15212
17:10:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.892689 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/master-2/wal/instance
17:10:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.909382 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } has no permanent_uuid. Determining permanent_uuid...
17:10:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.942798 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } has no permanent_uuid. Determining permanent_uuid...
17:10:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.952067 15289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } has no permanent_uuid. Determining permanent_uuid...
17:10:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.968489 15289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff: Bootstrap starting.
17:10:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.975513 15289 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff: Neither blocks nor log segments found. Creating new log.
17:10:51.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.977255 15289 log.cc:824] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff: Log is configured to *not* fsync() on all Append() calls
17:10:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:51.982549 15289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff: No bootstrap required, opened a new log
17:10:52.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.004599 15289 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.005414 15289 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
17:10:52.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.005606 15289 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3c3bad570dc04c148b8bfd262867b8ff, State: Initialized, Role: FOLLOWER
17:10:52.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.006306 15289 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.009124 15298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.010010 15298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: This master's current role is: FOLLOWER
17:10:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.010418 15289 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: configured and running, proceeding with master startup.
17:10:52.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:52.042434 15309 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:10:52.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:52.042820 15309 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:10:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.036418 15212 master_runner.cc:186] Error getting master registration for 127.14.179.254:39071: OK, instance_id { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" instance_seqno: 1714497048436975 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
17:10:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.061071 15212 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:10:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:52.061012 15243 tablet.cc:2369] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff: Can't schedule compaction. Clean time has not been advanced past its initial value.
17:10:52.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.066190 15135 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 6
17:10:52.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.083971 15135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: Bootstrap starting.
17:10:52.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.089448 15135 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: Neither blocks nor log segments found. Creating new log.
17:10:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.091248 15135 log.cc:824] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: Log is configured to *not* fsync() on all Append() calls
17:10:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.096393 15135 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: No bootstrap required, opened a new log
17:10:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.118079 15135 raft_consensus.cc:348] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.119206 15135 raft_consensus.cc:374] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
17:10:52.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.119786 15135 raft_consensus.cc:729] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b7ac01b86aa543249d04624e19665c02, State: Initialized, Role: FOLLOWER
17:10:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.120839 15135 consensus_queue.cc:260] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.124621 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.125253 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [sys.catalog]: This master's current role is: FOLLOWER
17:10:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.126758 15135 sys_catalog.cc:564] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [sys.catalog]: configured and running, proceeding with master startup.
17:10:52.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:52.150207 15089 tablet.cc:2369] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: Can't schedule compaction. Clean time has not been advanced past its initial value.
17:10:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:52.160423 15324 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:10:52.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:52.161132 15324 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:10:52.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.159968 15067 master_runner.cc:186] Error getting master registration for 127.14.179.253:36755: OK, instance_id { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" instance_seqno: 1714497050125754 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
17:10:52.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.198165 15067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:10:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.302417 15298 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:10:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.302968 15298 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.305917 15298 leader_election.cc:288] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b7ac01b86aa543249d04624e19665c02 (127.14.179.254:39071), e728f9b947d442b09957c7ce5a0514ee (127.14.179.253:36755)
17:10:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.307137 15110 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c3bad570dc04c148b8bfd262867b8ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b7ac01b86aa543249d04624e19665c02" is_pre_election: true
17:10:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.308125 15110 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3c3bad570dc04c148b8bfd262867b8ff in term 0.
17:10:52.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.309554 15240 leader_election.cc:302] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3c3bad570dc04c148b8bfd262867b8ff, b7ac01b86aa543249d04624e19665c02; no voters:
17:10:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.311129 15298 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Leader pre-election won for term 1
17:10:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.311532 15298 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
17:10:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.311837 15298 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 0 FOLLOWER]: Advancing to term 1
17:10:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.318984 15298 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.322023 15110 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c3bad570dc04c148b8bfd262867b8ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b7ac01b86aa543249d04624e19665c02"
17:10:52.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.322636 15110 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [term 0 FOLLOWER]: Advancing to term 1
17:10:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.334373 15110 raft_consensus.cc:2454] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3c3bad570dc04c148b8bfd262867b8ff in term 1.
17:10:52.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.335162 15298 leader_election.cc:288] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [CANDIDATE]: Term 1 election: Requested vote from peers b7ac01b86aa543249d04624e19665c02 (127.14.179.254:39071), e728f9b947d442b09957c7ce5a0514ee (127.14.179.253:36755)
17:10:52.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.342037 15240 leader_election.cc:302] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3c3bad570dc04c148b8bfd262867b8ff, b7ac01b86aa543249d04624e19665c02; no voters:
17:10:52.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.349822 15298 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 1 FOLLOWER]: Leader election won for term 1
17:10:52.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.358497 15298 raft_consensus.cc:686] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [term 1 LEADER]: Becoming Leader. State: Replica: 3c3bad570dc04c148b8bfd262867b8ff, State: Running, Role: LEADER
17:10:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.359604 15298 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.386538 15326 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3c3bad570dc04c148b8bfd262867b8ff. Latest consensus state: current_term: 1 leader_uuid: "3c3bad570dc04c148b8bfd262867b8ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.388576 15326 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: This master's current role is: LEADER
17:10:52.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.393107 15328 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
17:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.400853 15328 catalog_manager.cc:1484] Initializing Kudu cluster ID...
17:10:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.428169 15110 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [term 1 FOLLOWER]: Refusing update from remote peer 3c3bad570dc04c148b8bfd262867b8ff: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
17:10:52.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.430577 15298 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [LEADER]: Connected to new peer: Peer: permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
17:10:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.463595 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3c3bad570dc04c148b8bfd262867b8ff. Latest consensus state: current_term: 1 leader_uuid: "3c3bad570dc04c148b8bfd262867b8ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.464354 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [sys.catalog]: This master's current role is: FOLLOWER
17:10:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.486994 15329 mvcc.cc:205] Tried to move back new op lower bound from 7022579926715539456 to 7022579926475845632. Current Snapshot: MvccSnapshot[applied={T|T < 7022579926715539456}]
17:10:52.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.493630 15330 mvcc.cc:205] Tried to move back new op lower bound from 7022579926715539456 to 7022579926475845632. Current Snapshot: MvccSnapshot[applied={T|T < 7022579926715539456}]
17:10:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.498492 15298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3c3bad570dc04c148b8bfd262867b8ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.499949 15298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: This master's current role is: LEADER
17:10:52.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.506628 15210 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } attempt: 7
17:10:52.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.509884 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3c3bad570dc04c148b8bfd262867b8ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.510859 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02 [sys.catalog]: This master's current role is: FOLLOWER
17:10:52.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.514107 15328 catalog_manager.cc:1347] Generated new cluster ID: c046f932a9af49d5a2af8beee0bf669d
17:10:52.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.514425 15328 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
17:10:52.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.547488 15328 catalog_manager.cc:1370] Generated new certificate authority record
17:10:52.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.550424 15328 catalog_manager.cc:1504] Loading token signing keys...
17:10:52.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.559393 15210 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee: Bootstrap starting.
17:10:52.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.577062 15210 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee: Neither blocks nor log segments found. Creating new log.
17:10:52.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.584251 15210 log.cc:824] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee: Log is configured to *not* fsync() on all Append() calls
17:10:52.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.584530 15328 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff: Generated new TSK 0
17:10:52.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.585646 15328 catalog_manager.cc:1514] Initializing in-progress tserver states...
17:10:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.608610 15210 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee: No bootstrap required, opened a new log
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.620657 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/wal
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/logs
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data/info.pb
17:10:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.193:0
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.193
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.193
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
17:10:52.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.651121 15210 raft_consensus.cc:348] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.652451 15210 raft_consensus.cc:374] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
17:10:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.652994 15210 raft_consensus.cc:729] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e728f9b947d442b09957c7ce5a0514ee, State: Initialized, Role: FOLLOWER
17:10:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.654122 15210 consensus_queue.cc:260] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } }
17:10:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.660477 15210 sys_catalog.cc:564] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [sys.catalog]: configured and running, proceeding with master startup.
17:10:52.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.660635 15184 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c3bad570dc04c148b8bfd262867b8ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e728f9b947d442b09957c7ce5a0514ee"
17:10:52.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.661358 15184 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [term 0 FOLLOWER]: Advancing to term 1
17:10:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.664471 15185 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3c3bad570dc04c148b8bfd262867b8ff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e728f9b947d442b09957c7ce5a0514ee" is_pre_election: true
17:10:52.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.674800 15184 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3c3bad570dc04c148b8bfd262867b8ff in term 1.
17:10:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.677379 15183 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [term 1 FOLLOWER]: Refusing update from remote peer 3c3bad570dc04c148b8bfd262867b8ff: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
17:10:52.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.684015 15298 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [LEADER]: Connected to new peer: Peer: permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
17:10:52.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.659524 15337 sys_catalog.cc:455] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.689671 15337 sys_catalog.cc:458] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [sys.catalog]: This master's current role is: FOLLOWER
17:10:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.775831 15339 sys_catalog.cc:455] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3c3bad570dc04c148b8bfd262867b8ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.775569 15337 sys_catalog.cc:455] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3c3bad570dc04c148b8bfd262867b8ff. Latest consensus state: current_term: 1 leader_uuid: "3c3bad570dc04c148b8bfd262867b8ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.776669 15339 sys_catalog.cc:458] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [sys.catalog]: This master's current role is: FOLLOWER
17:10:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.776727 15337 sys_catalog.cc:458] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee [sys.catalog]: This master's current role is: FOLLOWER
17:10:52.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:52.783890 15206 debug-util.cc:398] Leaking SignalData structure 0x7b080003ad60 after lost signal to thread 15141
17:10:52.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:52.880395 15353 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
17:10:53.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.157879 15298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3c3bad570dc04c148b8bfd262867b8ff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ac01b86aa543249d04624e19665c02" member_type: VOTER last_known_addr { host: "127.14.179.254" port: 39071 } } peers { permanent_uuid: "e728f9b947d442b09957c7ce5a0514ee" member_type: VOTER last_known_addr { host: "127.14.179.253" port: 36755 } } peers { permanent_uuid: "3c3bad570dc04c148b8bfd262867b8ff" member_type: VOTER last_known_addr { host: "127.14.179.252" port: 34823 } } }
17:10:53.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.159080 15298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3c3bad570dc04c148b8bfd262867b8ff [sys.catalog]: This master's current role is: LEADER
17:10:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.167766 15324 catalog_manager.cc:1259] Loaded cluster ID: c046f932a9af49d5a2af8beee0bf669d
17:10:53.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.168375 15324 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: loading cluster ID for follower catalog manager: success
17:10:53.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.187016 15324 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: acquiring CA information for follower catalog manager: success
17:10:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.193032 15324 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b7ac01b86aa543249d04624e19665c02: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
17:10:53.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:53.239254 15336 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:10:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:53.240298 15336 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:10:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:53.241848 15336 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:10:53.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:53.299547 15336 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:10:53.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:53.301385 15336 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.179.193
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.365614 15336 tablet_server_runner.cc:78] Tablet server non-default flags:
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/wal
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.193:0
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data/info.pb
17:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.193
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.193
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/logs
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:10:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:10:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.369146 15336 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:10:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.371692 15336 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:10:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:53.419567 15363 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:53.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.898061 15353 catalog_manager.cc:1259] Loaded cluster ID: c046f932a9af49d5a2af8beee0bf669d
17:10:53.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.898330 15353 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee: loading cluster ID for follower catalog manager: success
17:10:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.918330 15353 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee: acquiring CA information for follower catalog manager: success
17:10:53.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:53.923945 15353 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e728f9b947d442b09957c7ce5a0514ee: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
17:10:56.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:53.420764 15362 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:56.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:54.794296 15361 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 15336
17:10:56.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:56.443125 15361 kernel_stack_watchdog.cc:198] Thread 15336 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@3/src/kudu/util/thread.cc:637 for 375ms:
17:10:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
17:10:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
17:10:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
17:10:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
17:10:56.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:56.447465 15336 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.031s user 0.645s sys 1.103s
17:10:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:56.448223 15336 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.032s user 0.645s sys 1.104s
17:10:56.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:56.449429 15364 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
17:10:56.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:56.450582 15366 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:56.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.451236 15336 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
17:10:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.453333 15336 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:10:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.467391 15336 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:10:56.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.469214 15336 hybrid_clock.cc:648] HybridClock initialized: now 1714497056469154 us; error 72 us; skew 500 ppm
17:10:56.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.470558 15336 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:10:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.500862 15336 webserver.cc:460] Webserver started at http://127.14.179.193:40497/ using document root <none> and password file <none>
17:10:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.507766 15336 fs_manager.cc:359] Metadata directory not provided
17:10:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.508482 15336 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:10:56.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.509748 15336 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:10:56.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.517063 15336 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data/instance:
17:10:56.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d83e0bbccd6143099efe6fd403770a7d"
17:10:56.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:56 on dist-test-slave-2p34"
17:10:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.518942 15336 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/wal/instance:
17:10:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d83e0bbccd6143099efe6fd403770a7d"
17:10:56.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:56 on dist-test-slave-2p34"
17:10:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.534608 15336 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.004s sys 0.008s
17:10:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.545816 15374 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:10:56.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.547447 15336 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.003s sys 0.003s
17:10:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.548115 15336 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/wal
17:10:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d83e0bbccd6143099efe6fd403770a7d"
17:10:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:56 on dist-test-slave-2p34"
17:10:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.548916 15336 fs_report.cc:383] FS layout report
17:10:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:10:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/wal
17:10:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/wal
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data/data
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:10:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:10:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.651062 15336 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:10:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.653587 15336 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:10:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.654490 15336 kserver.cc:163] Server-wide thread pool size limit: 3276
17:10:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.672674 15336 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
17:10:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.681896 15336 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
17:10:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.682579 15336 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.001s
17:10:56.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.683113 15336 ts_tablet_manager.cc:590] Registered 0 tablets
17:10:56.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:56.683568 15336 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
17:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.078800 15336 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.193:37477
17:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.078958 15488 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.193:37477 every 8 connection(s)
17:10:57.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.083400 15336 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/data/info.pb
17:10:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.084352 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15336
17:10:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.085000 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-0/wal/instance
17:10:57.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.101341 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/wal
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/logs
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data/info.pb
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.194:0
17:10:57.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.194
17:10:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.194
17:10:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:57.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
17:10:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.162422 15489 heartbeater.cc:347] Connected to a master server at 127.14.179.253:36755
17:10:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.162909 15489 heartbeater.cc:464] Registering TS with master...
17:10:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.164184 15489 heartbeater.cc:511] Master 127.14.179.253:36755 requested a full tablet report, sending...
17:10:57.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.167986 15175 ts_manager.cc:194] Registered new tserver with Master: d83e0bbccd6143099efe6fd403770a7d (127.14.179.193:37477)
17:10:57.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.172886 15493 heartbeater.cc:347] Connected to a master server at 127.14.179.254:39071
17:10:57.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.173280 15493 heartbeater.cc:464] Registering TS with master...
17:10:57.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.174253 15493 heartbeater.cc:511] Master 127.14.179.254:39071 requested a full tablet report, sending...
17:10:57.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.174661 15491 heartbeater.cc:347] Connected to a master server at 127.14.179.252:34823
17:10:57.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.175002 15491 heartbeater.cc:464] Registering TS with master...
17:10:57.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.175925 15491 heartbeater.cc:511] Master 127.14.179.252:34823 requested a full tablet report, sending...
17:10:57.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.177605 15100 ts_manager.cc:194] Registered new tserver with Master: d83e0bbccd6143099efe6fd403770a7d (127.14.179.193:37477)
17:10:57.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.178376 15254 ts_manager.cc:194] Registered new tserver with Master: d83e0bbccd6143099efe6fd403770a7d (127.14.179.193:37477)
17:10:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.180522 15254 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.179.193:54627
17:10:57.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.479437 15496 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:10:57.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.480006 15496 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:10:57.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.480633 15496 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:10:57.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.512442 15496 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:10:57.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.513403 15496 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.179.194
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.547645 15496 tablet_server_runner.cc:78] Tablet server non-default flags:
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/wal
17:10:57.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.194:0
17:10:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:10:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data/info.pb
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.194
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.194
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/logs
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:10:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.549140 15496 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:10:57.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.551299 15496 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:10:57.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.572126 15506 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:58.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.183435 15491 heartbeater.cc:503] Master 127.14.179.252:34823 was elected leader, sending a full tablet report...
17:10:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.573307 15508 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:57.574409 15505 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:10:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:57.574755 15496 server_base.cc:1034] running on GCE node
17:10:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.695523 15496 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:10:58.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.698357 15496 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:10:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.699791 15496 hybrid_clock.cc:648] HybridClock initialized: now 1714497058699744 us; error 59 us; skew 500 ppm
17:10:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.700567 15496 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.708248 15496 webserver.cc:460] Webserver started at http://127.14.179.194:46745/ using document root <none> and password file <none>
17:10:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.709159 15496 fs_manager.cc:359] Metadata directory not provided
17:10:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.709354 15496 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:10:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.709762 15496 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:10:58.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.714685 15496 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data/instance:
17:10:58.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01afdc79ea034a9ba9e7af51e28edfac"
17:10:58.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:58 on dist-test-slave-2p34"
17:10:58.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.715812 15496 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/wal/instance:
17:10:58.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01afdc79ea034a9ba9e7af51e28edfac"
17:10:58.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:58 on dist-test-slave-2p34"
17:10:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.722707 15496 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
17:10:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.728930 15515 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:10:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.730152 15496 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
17:10:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.730569 15496 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/wal
17:10:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "01afdc79ea034a9ba9e7af51e28edfac"
17:10:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:10:58 on dist-test-slave-2p34"
17:10:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.730943 15496 fs_report.cc:383] FS layout report
17:10:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:10:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/wal
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/wal
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data/data
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:10:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:10:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.798815 15496 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:10:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.800498 15496 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:10:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.800923 15496 kserver.cc:163] Server-wide thread pool size limit: 3276
17:10:58.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.803366 15496 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
17:10:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.807463 15496 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
17:10:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.807653 15496 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
17:10:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.807847 15496 ts_tablet_manager.cc:590] Registered 0 tablets
17:10:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.807984 15496 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
17:10:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.961133 15496 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.194:33919
17:10:58.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.961802 15628 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.194:33919 every 8 connection(s)
17:10:58.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.964032 15496 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/data/info.pb
17:10:58.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.964672 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15496
17:10:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.965102 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-1/wal/instance
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:58.980427 15055 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/wal
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/logs
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data/info.pb
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.195:0
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.195
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.195
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
17:10:59.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.030462 15630 heartbeater.cc:347] Connected to a master server at 127.14.179.252:34823
17:10:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.030946 15630 heartbeater.cc:464] Registering TS with master...
17:10:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.031334 15629 heartbeater.cc:347] Connected to a master server at 127.14.179.253:36755
17:10:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.031687 15629 heartbeater.cc:464] Registering TS with master...
17:10:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.032192 15630 heartbeater.cc:511] Master 127.14.179.252:34823 requested a full tablet report, sending...
17:10:59.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.032698 15629 heartbeater.cc:511] Master 127.14.179.253:36755 requested a full tablet report, sending...
17:10:59.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.034838 15254 ts_manager.cc:194] Registered new tserver with Master: 01afdc79ea034a9ba9e7af51e28edfac (127.14.179.194:33919)
17:10:59.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.035790 15175 ts_manager.cc:194] Registered new tserver with Master: 01afdc79ea034a9ba9e7af51e28edfac (127.14.179.194:33919)
17:10:59.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.036623 15634 heartbeater.cc:347] Connected to a master server at 127.14.179.254:39071
17:10:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.036974 15634 heartbeater.cc:464] Registering TS with master...
17:10:59.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.036939 15254 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.179.194:58011
17:10:59.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.037894 15634 heartbeater.cc:511] Master 127.14.179.254:39071 requested a full tablet report, sending...
17:10:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.040194 15100 ts_manager.cc:194] Registered new tserver with Master: 01afdc79ea034a9ba9e7af51e28edfac (127.14.179.194:33919)
17:10:59.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.319968 15633 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:10:59.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.320439 15633 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:10:59.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.320863 15633 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:10:59.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.352133 15633 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:10:59.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.352922 15633 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.179.195
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.387144 15633 tablet_server_runner.cc:78] Tablet server non-default flags:
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.179.212:45155
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/wal
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.179.195:0
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:10:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data/info.pb
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.179.195
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.179.254:39071,127.14.179.253:36755,127.14.179.252:34823
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.179.195
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/logs
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:10:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.388536 15633 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:10:59.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.390282 15633 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:10:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.409574 15648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.040758 15630 heartbeater.cc:503] Master 127.14.179.252:34823 was elected leader, sending a full tablet report...
17:11:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.410481 15646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:10:59.411550 15633 server_base.cc:1034] running on GCE node
17:11:00.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:10:59.410746 15645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:00.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.522670 15633 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:11:00.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.525578 15633 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:11:00.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.527027 15633 hybrid_clock.cc:648] HybridClock initialized: now 1714497060526982 us; error 61 us; skew 500 ppm
17:11:00.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.527827 15633 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:11:00.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.534493 15633 webserver.cc:460] Webserver started at http://127.14.179.195:43685/ using document root <none> and password file <none>
17:11:00.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.535399 15633 fs_manager.cc:359] Metadata directory not provided
17:11:00.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.535598 15633 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:11:00.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.535986 15633 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:11:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.540796 15633 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data/instance:
17:11:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d50f901559914b5b8ba5ca2045d1eef3"
17:11:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:00 on dist-test-slave-2p34"
17:11:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.541812 15633 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/wal/instance:
17:11:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d50f901559914b5b8ba5ca2045d1eef3"
17:11:00.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:00 on dist-test-slave-2p34"
17:11:00.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.548735 15633 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
17:11:00.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.553999 15656 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:11:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.554991 15633 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
17:11:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.555310 15633 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/wal
17:11:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d50f901559914b5b8ba5ca2045d1eef3"
17:11:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:00 on dist-test-slave-2p34"
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.555698 15633 fs_report.cc:383] FS layout report
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/wal
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/wal
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data/data
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:11:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:11:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.628649 15633 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:11:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.630175 15633 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:11:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.630661 15633 kserver.cc:163] Server-wide thread pool size limit: 3276
17:11:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.633215 15633 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
17:11:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.637815 15633 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
17:11:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.638084 15633 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
17:11:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.638393 15633 ts_tablet_manager.cc:590] Registered 0 tablets
17:11:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.638571 15633 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
17:11:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.792464 15633 rpc_server.cc:307] RPC server started. Bound to: 127.14.179.195:45069
17:11:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.792583 15768 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.14.179.195:45069 every 8 connection(s)
17:11:00.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.795063 15633 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/data/info.pb
17:11:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.800690 15055 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15633
17:11:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.801292 15055 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster1315050811298256656/ts-2/wal/instance
17:11:00.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.837872 15769 heartbeater.cc:347] Connected to a master server at 127.14.179.253:36755
17:11:00.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.838382 15769 heartbeater.cc:464] Registering TS with master...
17:11:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.839736 15769 heartbeater.cc:511] Master 127.14.179.253:36755 requested a full tablet report, sending...
17:11:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.839959 15770 heartbeater.cc:347] Connected to a master server at 127.14.179.252:34823
17:11:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.840277 15770 heartbeater.cc:464] Registering TS with master...
17:11:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.841132 15770 heartbeater.cc:511] Master 127.14.179.252:34823 requested a full tablet report, sending...
17:11:00.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.842774 15175 ts_manager.cc:194] Registered new tserver with Master: d50f901559914b5b8ba5ca2045d1eef3 (127.14.179.195:45069)
17:11:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.843672 15254 ts_manager.cc:194] Registered new tserver with Master: d50f901559914b5b8ba5ca2045d1eef3 (127.14.179.195:45069)
17:11:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.845815 15254 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.179.195:34295
17:11:00.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.852423 15773 heartbeater.cc:347] Connected to a master server at 127.14.179.254:39071
17:11:00.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.852730 15773 heartbeater.cc:464] Registering TS with master...
17:11:00.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.853322 15773 heartbeater.cc:511] Master 127.14.179.254:39071 requested a full tablet report, sending...
17:11:00.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.855278 15100 ts_manager.cc:194] Registered new tserver with Master: d50f901559914b5b8ba5ca2045d1eef3 (127.14.179.195:45069)
17:11:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:00.865013 15055 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
17:11:00.875 [DEBUG - main] (MiniKuduCluster.java:195) Response:
17:11:00.876 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
17:11:00.887 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.14.179.254"
port: 39071
}
bound_http_address {
host: "127.14.179.254"
port: 35071
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.14.179.253"
port: 36755
}
bound_http_address {
host: "127.14.179.253"
port: 39023
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.14.179.252"
port: 34823
}
bound_http_address {
host: "127.14.179.252"
port: 38253
}
}
}
17:11:00.893 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
17:11:00.897 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.14.179.193"
port: 37477
}
bound_http_address {
host: "127.14.179.193"
port: 40497
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.14.179.194"
port: 33919
}
bound_http_address {
host: "127.14.179.194"
port: 46745
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.14.179.195"
port: 45069
}
bound_http_address {
host: "127.14.179.195"
port: 43685
}
}
}
17:11:00.897 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
17:11:01.092 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
17:11:01.214 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNB5BOS/test-tmp
17:11:01.215 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
17:11:01.215 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNB5BOS/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
17:11:01.218 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
17:11:01.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:01.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
17:11:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:01.581372 15780 mini_oidc.cc:152] Starting JWKS server
17:11:01.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:01.585227 15780 webserver.cc:460] Webserver started at http://127.0.0.1:39311/ using document root <none> and password file <none>
17:11:01.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:01.585647 15780 mini_oidc.cc:189] Starting OIDC Discovery server
17:11:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:01.587266 15780 webserver.cc:460] Webserver started at http://127.0.0.1:36085/ using document root <none> and password file <none>
17:11:01.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
17:11:01.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/principal' for realm 'KRBTEST.COM',
17:11:01.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
17:11:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:01 dist-test-slave-2p34 krb5kdc[15792](info): setting up network...
17:11:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
17:11:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:01 dist-test-slave-2p34 krb5kdc[15792](info): set up 2 sockets
17:11:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:01 dist-test-slave-2p34 krb5kdc[15792](info): commencing operation
17:11:01.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
17:11:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:01.849752 15770 heartbeater.cc:503] Master 127.14.179.252:34823 was elected leader, sending a full tablet report...
17:11:02.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:02.625634 15780 mini_kdc.cc:121] Time spent starting KDC: real 0.998s user 0.005s sys 0.015s
17:11:02.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
17:11:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
17:11:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
17:11:02.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
17:11:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
17:11:02.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
17:11:02.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
17:11:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
17:11:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
17:11:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
17:11:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/test-user.keytab.
17:11:02.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/test-user.keytab.
17:11:02.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:02 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497062, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
17:11:02.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T17:11:02Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
17:11:02.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T17:11:02Z Disabled control of system clock
17:11:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.105.62@KRBTEST.COM; defaulting to no policy
17:11:02.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
17:11:02.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.105.62@KRBTEST.COM" created.
17:11:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
17:11:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.62.keytab.
17:11:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.62.keytab.
17:11:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:02.816547 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data
17:11:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/logs
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/info.pb
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:45647
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.62.keytab
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
17:11:03.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.135504 15805 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:11:03.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.136113 15805 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:11:03.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.136682 15805 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:11:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.168799 15805 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
17:11:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.169077 15805 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:11:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.169355 15805 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:11:03.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.169701 15805 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:11:03.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.170027 15805 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:11:03.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.170389 15805 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:11:03.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.170653 15805 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39311/jwks.json
17:11:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:03.208603 15805 master_runner.cc:386] Master server non-default flags:
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.62.keytab
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:45647
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/info.pb
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/logs
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:11:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:03.209707 15805 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:11:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:03.210713 15805 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:11:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.225806 15812 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.226725 15813 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:03.227547 15815 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:03.228165 15805 server_base.cc:1034] running on GCE node
17:11:04.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.376927 15805 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:11:04.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.379845 15805 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:11:04.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.381182 15805 hybrid_clock.cc:648] HybridClock initialized: now 1714497064381112 us; error 94 us; skew 500 ppm
17:11:04.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.381927 15805 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:11:04.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:04 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497064, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.387449 15805 init.cc:377] Logged in from keytab as kudu/127.15.105.62@KRBTEST.COM (short username kudu)
17:11:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.393337 15805 webserver.cc:460] Webserver started at http://127.15.105.62:42527/ using document root <none> and password file <none>
17:11:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.394279 15805 fs_manager.cc:359] Metadata directory not provided
17:11:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.394524 15805 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:11:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.394999 15805 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:11:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.400193 15805 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/instance:
17:11:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd0d44b5c828438080d5a10cae87d03c"
17:11:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:04 on dist-test-slave-2p34"
17:11:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.401441 15805 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal/instance:
17:11:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd0d44b5c828438080d5a10cae87d03c"
17:11:04.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:04 on dist-test-slave-2p34"
17:11:04.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.409022 15805 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
17:11:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.415316 15824 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:11:04.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.416390 15805 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.416752 15805 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd0d44b5c828438080d5a10cae87d03c"
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:04 on dist-test-slave-2p34"
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.417104 15805 fs_report.cc:383] FS layout report
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/data
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:11:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:11:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:11:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:11:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:11:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:11:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:11:04.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.474579 15805 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:11:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.480221 15805 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:11:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.481186 15805 kserver.cc:163] Server-wide thread pool size limit: 3276
17:11:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.562222 15805 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.62:45647
17:11:04.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.562295 15875 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.62:45647 every 8 connection(s)
17:11:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.564946 15805 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/info.pb
17:11:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.568310 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15805
17:11:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.568827 15780 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal/instance
17:11:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.571813 15876 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
17:11:04.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.105.61@KRBTEST.COM; defaulting to no policy
17:11:04.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.585888 15876 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } has no permanent_uuid. Determining permanent_uuid...
17:11:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
17:11:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.105.61@KRBTEST.COM" created.
17:11:04.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:04 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497064, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
17:11:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
17:11:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.61.keytab.
17:11:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.61.keytab.
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.616271 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/logs
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/info.pb
17:11:04.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41549
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.61.keytab
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:04.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
17:11:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.622833 15876 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } has no permanent_uuid. Determining permanent_uuid...
17:11:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.625752 15827 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.630525 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41549: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.680140 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } attempt: 1
17:11:04.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.684499 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41549: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.752914 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } attempt: 2
17:11:04.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.756927 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41549: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:04.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.848381 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } attempt: 3
17:11:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.852985 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41549: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:04.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.950331 15882 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:11:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.951015 15882 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:11:04.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.951778 15882 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:11:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.983135 15882 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
17:11:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.983438 15882 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:11:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.983815 15882 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:11:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.984110 15882 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:11:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.984407 15882 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:11:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.984689 15882 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:11:04.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:04.984898 15882 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39311/jwks.json
17:11:04.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:04.996382 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } attempt: 4
17:11:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:05.000049 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41549: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:05.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:05.021338 15882 master_runner.cc:386] Master server non-default flags:
17:11:05.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:05.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.61.keytab
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41549
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/info.pb
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/logs
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:11:05.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:11:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:11:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:11:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:11:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:05.022862 15882 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:11:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:05.024241 15882 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:11:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:05.040812 15889 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:05.299432 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } attempt: 5
17:11:05.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:05.303850 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41549: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:05.851452 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } attempt: 6
17:11:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:05.861869 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.61:41549: Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:05.041626 15890 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:06.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:05.044021 15882 server_base.cc:1034] running on GCE node
17:11:06.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:05.045199 15892 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:06.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.192452 15882 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:11:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.195317 15882 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:11:06.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.196729 15882 hybrid_clock.cc:648] HybridClock initialized: now 1714497066196684 us; error 54 us; skew 500 ppm
17:11:06.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.197549 15882 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:11:06.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:06 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497066, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.204197 15882 init.cc:377] Logged in from keytab as kudu/127.15.105.61@KRBTEST.COM (short username kudu)
17:11:06.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.210278 15882 webserver.cc:460] Webserver started at http://127.15.105.61:35057/ using document root <none> and password file <none>
17:11:06.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.211262 15882 fs_manager.cc:359] Metadata directory not provided
17:11:06.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.211522 15882 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:11:06.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.212030 15882 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:11:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.217476 15882 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/instance:
17:11:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbed83708289438c9a983ff8579912be"
17:11:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:06 on dist-test-slave-2p34"
17:11:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.218619 15882 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal/instance:
17:11:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbed83708289438c9a983ff8579912be"
17:11:06.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:06 on dist-test-slave-2p34"
17:11:06.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.226315 15882 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.010s sys 0.000s
17:11:06.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.232146 15902 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:11:06.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.233325 15882 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.233716 15882 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbed83708289438c9a983ff8579912be"
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:06 on dist-test-slave-2p34"
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.234066 15882 fs_report.cc:383] FS layout report
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/data
17:11:06.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:11:06.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:11:06.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.293573 15882 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:11:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.299400 15882 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:11:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.299906 15882 kserver.cc:163] Server-wide thread pool size limit: 3276
17:11:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.379562 15882 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.61:41549
17:11:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.379635 15953 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.61:41549 every 8 connection(s)
17:11:06.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.382133 15882 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/info.pb
17:11:06.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.382967 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15882
17:11:06.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.383402 15780 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal/instance
17:11:06.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.389112 15954 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
17:11:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.105.60@KRBTEST.COM; defaulting to no policy
17:11:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.400605 15954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } has no permanent_uuid. Determining permanent_uuid...
17:11:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
17:11:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.105.60@KRBTEST.COM" created.
17:11:06.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:06 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497066, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
17:11:06.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
17:11:06.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.60.keytab.
17:11:06.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.105.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.60.keytab.
17:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.430522 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/logs
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/info.pb
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:40151
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.60.keytab
17:11:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
17:11:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
17:11:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
17:11:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
17:11:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.435629 15954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } has no permanent_uuid. Determining permanent_uuid...
17:11:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:06 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497066, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
17:11:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.462654 15954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } has no permanent_uuid. Determining permanent_uuid...
17:11:06.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.465916 15903 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.470780 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.520293 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 1
17:11:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.524121 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:06.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.592607 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 2
17:11:06.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.596385 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:06.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.687849 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 3
17:11:06.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.691427 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.752789 15960 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:11:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.753408 15960 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:11:06.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.753993 15960 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:11:06.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.788625 15960 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
17:11:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.788856 15960 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:11:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.789121 15960 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:11:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.789321 15960 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:11:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.789568 15960 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:11:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.789777 15960 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:11:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.790129 15960 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39311/jwks.json
17:11:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.826364 15960 master_runner.cc:386] Master server non-default flags:
17:11:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.60.keytab
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:40151
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/info.pb
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:11:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/logs
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.827561 15960 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:11:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.828567 15960 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:11:06.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.834862 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 4
17:11:06.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.840410 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:06.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.846093 15969 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:06.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.922468 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } attempt: 7
17:11:06.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:06 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497064, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
17:11:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.944689 15876 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } has no permanent_uuid. Determining permanent_uuid...
17:11:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.950984 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:07.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:07.009375 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 1
17:11:07.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:07.014434 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:07.095808 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 2
17:11:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:07.100914 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:07.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:07.139853 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 5
17:11:07.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:07.143987 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:07.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:07.186343 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 3
17:11:07.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:07.191807 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:07.332223 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 4
17:11:07.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:07.335865 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:07.619520 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 5
17:11:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:07.623440 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:07.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:07.691619 15954 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 6
17:11:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:07.698027 15954 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.846691 15970 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:06.848191 15960 server_base.cc:1034] running on GCE node
17:11:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:06.847743 15972 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:08.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.018508 15960 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:11:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.021751 15960 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:11:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.023130 15960 hybrid_clock.cc:648] HybridClock initialized: now 1714497068023102 us; error 35 us; skew 500 ppm
17:11:08.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.024020 15960 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:11:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497068, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.029752 15960 init.cc:377] Logged in from keytab as kudu/127.15.105.60@KRBTEST.COM (short username kudu)
17:11:08.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.035027 15960 webserver.cc:460] Webserver started at http://127.15.105.60:42067/ using document root <none> and password file <none>
17:11:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.036007 15960 fs_manager.cc:359] Metadata directory not provided
17:11:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.036235 15960 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:11:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.036754 15960 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
17:11:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.041684 15960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/instance:
17:11:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bb534d8de834a17b9d07567f42e6336"
17:11:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:08 on dist-test-slave-2p34"
17:11:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.042830 15960 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal/instance:
17:11:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bb534d8de834a17b9d07567f42e6336"
17:11:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:08 on dist-test-slave-2p34"
17:11:08.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.049937 15960 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
17:11:08.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.056046 15986 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:11:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.057157 15960 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
17:11:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.057513 15960 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bb534d8de834a17b9d07567f42e6336"
17:11:08.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:08 on dist-test-slave-2p34"
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.057859 15960 fs_report.cc:383] FS layout report
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/data
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:11:08.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:11:08.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:11:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:11:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:11:08.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.121795 15960 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:11:08.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.128028 15960 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:11:08.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.128463 15960 kserver.cc:163] Server-wide thread pool size limit: 3276
17:11:08.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.176141 15876 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } attempt: 6
17:11:08.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.184227 15876 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.15.105.60:40151: Network error: Client connection negotiation failed: client connection to 127.15.105.60:40151: connect: Connection refused (error 111)
17:11:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.211045 15960 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.60:40151
17:11:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.211172 16039 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.60:40151 every 8 connection(s)
17:11:08.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.214751 15960 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/info.pb
17:11:08.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.221704 16040 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
17:11:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.221814 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 15960
17:11:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.222317 15780 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal/instance
17:11:08.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.236050 16040 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } has no permanent_uuid. Determining permanent_uuid...
17:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497062, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.60@KRBTEST.COM
17:11:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497068, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
17:11:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.271154 16040 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } has no permanent_uuid. Determining permanent_uuid...
17:11:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497068, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
17:11:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497062, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.61@KRBTEST.COM
17:11:08.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.294800 16040 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } has no permanent_uuid. Determining permanent_uuid...
17:11:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497068, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for kudu/127.15.105.60@KRBTEST.COM
17:11:08.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497062, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.62@KRBTEST.COM
17:11:08.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.323527 16040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Bootstrap starting.
17:11:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.325153 15780 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
17:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.330701 16040 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Neither blocks nor log segments found. Creating new log.
17:11:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.333123 16040 log.cc:824] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Log is configured to *not* fsync() on all Append() calls
17:11:08.336 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:08.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
17:11:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.338660 16040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: No bootstrap required, opened a new log
17:11:08.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
17:11:08.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
17:11:08.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.105.62"
port: 45647
}
bound_http_address {
host: "127.15.105.62"
port: 42527
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.105.61"
port: 41549
}
bound_http_address {
host: "127.15.105.61"
port: 35057
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.105.60"
port: 40151
}
bound_http_address {
host: "127.15.105.60"
port: 42067
}
}
}
17:11:08.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
17:11:08.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:08.352 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
17:11:08.353 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
17:11:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.362712 16040 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.363461 16040 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
17:11:08.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.363745 16040 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4bb534d8de834a17b9d07567f42e6336, State: Initialized, Role: FOLLOWER
17:11:08.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.364619 16040 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.368176 16050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } } }
17:11:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.368888 16050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [sys.catalog]: This master's current role is: FOLLOWER
17:11:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.369823 16040 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [sys.catalog]: configured and running, proceeding with master startup.
17:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497068, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
17:11:08.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.383932 15992 tablet.cc:2369] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Can't schedule compaction. Clean time has not been advanced past its initial value.
17:11:08.389 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:08.391 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc
17:11:08.398 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
17:11:08.400 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc
17:11:08.401 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
17:11:08.405 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
17:11:08.407 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
17:11:08.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
17:11:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.421583 16062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.421939 16062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:08.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.404186 15960 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: OK, instance_id { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" instance_seqno: 1714497064540446 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
17:11:08.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.439827 15960 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:08.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:08 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497068, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:08.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
17:11:08.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:08.454 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc
17:11:08.456 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
17:11:08.456 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.105.60:40151
17:11:08.457 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
17:11:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.458331 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15960
17:11:08.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:08.480 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.105.62:45647
17:11:08.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
17:11:08.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.480687 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15805
17:11:08.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:08.503 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.105.61:41549
17:11:08.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
17:11:08.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.503712 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15882
17:11:08.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:08.525 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.105.60:40151
17:11:08.527 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.528924 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/logs
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/info.pb
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:40151
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
17:11:08.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42067
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.60.keytab
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
17:11:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.843166 16067 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:11:08.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.843804 16067 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:11:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.844394 16067 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:11:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.877877 16067 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
17:11:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.878182 16067 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:11:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.878638 16067 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:11:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.878976 16067 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:11:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.879297 16067 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:11:08.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.879652 16067 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:11:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.879889 16067 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39311/jwks.json
17:11:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.921000 16067 master_runner.cc:386] Master server non-default flags:
17:11:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.60.keytab
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.60
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.60:40151
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/info.pb
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.60
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42067
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/logs
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:11:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:11:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:11:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:11:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:11:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.922468 16067 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:11:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.923893 16067 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:11:08.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.939301 16075 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:10.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.939932 16074 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:10.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:08.941198 16067 server_base.cc:1034] running on GCE node
17:11:10.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:08.940539 16077 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.097469 16067 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:11:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.100135 16067 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:11:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.101472 16067 hybrid_clock.cc:648] HybridClock initialized: now 1714497070101443 us; error 44 us; skew 500 ppm
17:11:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.102236 16067 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:11:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:10 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497070, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.113812 16067 init.cc:377] Logged in from keytab as kudu/127.15.105.60@KRBTEST.COM (short username kudu)
17:11:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.118278 16067 webserver.cc:460] Webserver started at http://127.15.105.60:42067/ using document root <none> and password file <none>
17:11:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.119163 16067 fs_manager.cc:359] Metadata directory not provided
17:11:10.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.119385 16067 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:11:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.127103 16067 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.003s
17:11:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.131392 16087 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:11:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.132476 16067 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.132755 16067 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bb534d8de834a17b9d07567f42e6336"
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:08 on dist-test-slave-2p34"
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.135010 16067 fs_report.cc:383] FS layout report
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/wal
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/data
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:11:10.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:11:10.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.184008 16067 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:11:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.189041 16067 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:11:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.189445 16067 kserver.cc:163] Server-wide thread pool size limit: 3276
17:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.287101 16067 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.60:40151
17:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.287225 16138 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.60:40151 every 8 connection(s)
17:11:10.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.289754 16067 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-2/data/info.pb
17:11:10.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.297742 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 16067
17:11:10.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:10.298 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.105.62:45647
17:11:10.298 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.300340 16139 sys_catalog.cc:263] Verifying existing consensus state
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.300449 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:11:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/logs
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/info.pb
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:45647
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42527
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.62.keytab
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
17:11:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
17:11:10.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.308437 16139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Bootstrap starting.
17:11:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.340574 16139 log.cc:824] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Log is configured to *not* fsync() on all Append() calls
17:11:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.345129 16139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
17:11:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.345825 16139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Bootstrap complete.
17:11:10.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.370165 16139 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:10.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.372699 16139 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
17:11:10.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.372900 16139 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4bb534d8de834a17b9d07567f42e6336, State: Initialized, Role: FOLLOWER
17:11:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.373641 16139 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.376438 16143 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } } }
17:11:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.377308 16143 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [sys.catalog]: This master's current role is: FOLLOWER
17:11:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.378029 16139 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [sys.catalog]: configured and running, proceeding with master startup.
17:11:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.419268 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:10.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.419991 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.414532 16067 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111),
17:11:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.444691 16067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.455024 16093 tablet.cc:2369] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: Can't schedule compaction. Clean time has not been advanced past its initial value.
17:11:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.593492 16143 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:11:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.594053 16143 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:10.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.597193 16143 leader_election.cc:288] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647), fbed83708289438c9a983ff8579912be (127.15.105.61:41549)
17:11:10.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.602481 16090 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.610005 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647): Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.610884 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fbed83708289438c9a983ff8579912be (127.15.105.61:41549): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.611400 16090 leader_election.cc:302] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bb534d8de834a17b9d07567f42e6336; no voters: bd0d44b5c828438080d5a10cae87d03c, fbed83708289438c9a983ff8579912be
17:11:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.612385 16143 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
17:11:10.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.648749 16141 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:11:10.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.649288 16141 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:11:10.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.649847 16141 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:11:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.681835 16141 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
17:11:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.682070 16141 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:11:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.682337 16141 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:11:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.682535 16141 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:11:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.682727 16141 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:11:10.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.682914 16141 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:11:10.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.683053 16141 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39311/jwks.json
17:11:10.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.719422 16141 master_runner.cc:386] Master server non-default flags:
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.62.keytab
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.62
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.62:45647
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/info.pb
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.62
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42527
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:11:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:11:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:11:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/logs
17:11:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:11:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.720952 16141 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:11:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.722298 16141 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:11:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.742301 16166 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:11.424412 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:11.424737 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:10.742954 16141 server_base.cc:1034] running on GCE node
17:11:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.743824 16165 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:10.744693 16168 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.934734 16141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:11:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.937813 16141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:11:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.939252 16141 hybrid_clock.cc:648] HybridClock initialized: now 1714497071939195 us; error 74 us; skew 500 ppm
17:11:11.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.940124 16141 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:11:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:11 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497071, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.945986 16141 init.cc:377] Logged in from keytab as kudu/127.15.105.62@KRBTEST.COM (short username kudu)
17:11:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.952123 16141 webserver.cc:460] Webserver started at http://127.15.105.62:42527/ using document root <none> and password file <none>
17:11:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.952947 16141 fs_manager.cc:359] Metadata directory not provided
17:11:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.953148 16141 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:11:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.961421 16141 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
17:11:11.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.966466 16177 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:11:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.967710 16141 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
17:11:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.968139 16141 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd0d44b5c828438080d5a10cae87d03c"
17:11:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:04 on dist-test-slave-2p34"
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:11.970391 16141 fs_report.cc:383] FS layout report
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/data
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:11:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:11:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.028285 16141 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:11:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.033932 16141 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:11:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.034466 16141 kserver.cc:163] Server-wide thread pool size limit: 3276
17:11:12.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.135094 16141 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.62:45647
17:11:12.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.135332 16228 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.62:45647 every 8 connection(s)
17:11:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.138183 16141 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/data/info.pb
17:11:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.144049 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 16141
17:11:12.145 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:12.145 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.105.61:41549
17:11:12.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.147904 15780 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/logs
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/info.pb
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41549
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.61.keytab
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
17:11:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.152716 16229 sys_catalog.cc:263] Verifying existing consensus state
17:11:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240430 17:11:12.153772 16229 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
17:11:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.154224 16141 master.cc:537] Master@127.15.105.62:45647 shutting down...
17:11:12.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.216851 16141 master.cc:552] Master@127.15.105.62:45647 shutdown complete.
17:11:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
17:11:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.427681 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.427922 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:12.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.508153 16231 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
17:11:12.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.508801 16231 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
17:11:12.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.509481 16231 flags.cc:425] Enabled unsafe flag: --never_fsync=true
17:11:12.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.541957 16231 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
17:11:12.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.542227 16231 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
17:11:12.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.542614 16231 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
17:11:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.542903 16231 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
17:11:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.543479 16231 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
17:11:12.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.544103 16231 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
17:11:12.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.544579 16231 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39311/jwks.json
17:11:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.580494 16231 master_runner.cc:386] Master server non-default flags:
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.105.20:33687
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.105.62:45647,127.15.105.61:41549,127.15.105.60:40151
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/krb5kdc/kudu_127.15.105.61.keytab
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.105.61
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.105.61:41549
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39311/jwks.json
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/info.pb
17:11:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.105.61
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35057
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/logs
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision ad3936521af034ffcac637f97cd8c932f6289b4f
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 30 Apr 2024 16:43:38 UTC on 1a15035a6830
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 60
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
17:11:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.581964 16231 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
17:11:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.583199 16231 file_cache.cc:492] Constructed file cache file cache with capacity 419430
17:11:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.588618 16236 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:11:12.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.589000 16236 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.592828 16236 leader_election.cc:288] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647), fbed83708289438c9a983ff8579912be (127.15.105.61:41549)
17:11:12.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.601933 16238 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:12.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.605525 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647): Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.605991 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fbed83708289438c9a983ff8579912be (127.15.105.61:41549): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.606274 16090 leader_election.cc:302] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bb534d8de834a17b9d07567f42e6336; no voters: bd0d44b5c828438080d5a10cae87d03c, fbed83708289438c9a983ff8579912be
17:11:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:12.607002 16236 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
17:11:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:13.433933 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:13.434497 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:13.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.606913 16244 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:12.603376 16240 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
17:11:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:13.740339 16241 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
17:11:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.740386 16231 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
17:11:13.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.744498 16231 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
17:11:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.747020 16231 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
17:11:13.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.748370 16231 hybrid_clock.cc:648] HybridClock initialized: now 1714497073748330 us; error 48 us; skew 500 ppm
17:11:13.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.749189 16231 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
17:11:13.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:13 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497073, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.754776 16231 init.cc:377] Logged in from keytab as kudu/127.15.105.61@KRBTEST.COM (short username kudu)
17:11:13.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.759990 16231 webserver.cc:460] Webserver started at http://127.15.105.61:35057/ using document root <none> and password file <none>
17:11:13.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.760918 16231 fs_manager.cc:359] Metadata directory not provided
17:11:13.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.761163 16231 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
17:11:13.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.769078 16231 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.003s
17:11:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.773234 16253 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
17:11:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.774209 16231 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
17:11:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.774564 16231 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data,/tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbed83708289438c9a983ff8579912be"
17:11:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-30 17:11:06 on dist-test-slave-2p34"
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.776836 16231 fs_report.cc:383] FS layout report
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/data
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
17:11:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
17:11:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
17:11:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
17:11:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
17:11:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
17:11:13.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.841471 16231 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
17:11:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.847052 16231 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
17:11:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.847611 16231 kserver.cc:163] Server-wide thread pool size limit: 3276
17:11:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.933648 16231 rpc_server.cc:307] RPC server started. Bound to: 127.15.105.61:41549
17:11:13.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.933792 16304 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.105.61:41549 every 8 connection(s)
17:11:13.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.936475 16231 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/data/info.pb
17:11:13.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.937217 15780 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu as pid 16231
17:11:13.937 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
17:11:13.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.946952 16305 sys_catalog.cc:263] Verifying existing consensus state
17:11:13.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240430 17:11:13.947664 16305 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
17:11:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:13.948002 16231 master.cc:537] Master@127.15.105.61:41549 shutting down...
17:11:13.994 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.105.60:40151(127.15.105.60:40151)] connecting to peer
17:11:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:14.042747 16231 master.cc:552] Master@127.15.105.61:41549 shutdown complete.
17:11:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNB5BOS/test-tmp/mini-kudu-cluster727715289649695411/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
17:11:14.067 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.075 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.106 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.105.60:40151(127.15.105.60:40151)] Successfully connected to peer
17:11:14.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.110 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.160 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.175 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
17:11:14.183 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.105.60
17:11:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:14 dist-test-slave-2p34 krb5kdc[15792](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497068, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.105.60@KRBTEST.COM
17:11:14.386 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd9b6bd95, L:/127.0.0.1:36358 - R:/127.15.105.60:40151] using SASL/GSSAPI
17:11:14.401 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.428 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.431 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.432 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:14.438619 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:14.438951 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:14.443 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.444 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.447 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.447 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.448 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.449 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.463 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.464 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.468 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.483 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.485 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.485 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.488 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.489 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.523 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.524 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.526 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.527 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.528 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.528 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.528 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.584 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.586 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.587 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.588 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.588 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.588 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.589 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.602 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.604 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.607 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.607 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.608 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.608 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.643 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:14.645 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:14.646 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.646 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.647 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:14.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:14.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:14.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:14.839519 16311 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:11:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:14.839916 16311 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:14.841492 16311 leader_election.cc:288] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647), fbed83708289438c9a983ff8579912be (127.15.105.61:41549)
17:11:14.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:14.849485 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647): Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:14.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:14.850013 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fbed83708289438c9a983ff8579912be (127.15.105.61:41549): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:14.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:14.850318 16090 leader_election.cc:302] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bb534d8de834a17b9d07567f42e6336; no voters: bd0d44b5c828438080d5a10cae87d03c, fbed83708289438c9a983ff8579912be
17:11:14.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:14.850925 16311 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
17:11:15.003 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:15.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:15.005 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.007 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.007 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.008 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.008 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:15.441745 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:15.442044 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:15.447381 16067 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111),
17:11:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:15.447726 16067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:15.663 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:15.664 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:15.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.667 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.667 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.667 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.668 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:15.822 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:15.824 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:15.825 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.825 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.825 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:15.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.826 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:15.826 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:16.323 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:16.324 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:16.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:16.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:16.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:16.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:16.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:16.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:16.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:16.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:16.444934 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:16.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:16.445192 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:17.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:17.118359 16317 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:11:17.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:17.118712 16317 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:17.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:17.120162 16317 leader_election.cc:288] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647), fbed83708289438c9a983ff8579912be (127.15.105.61:41549)
17:11:17.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:17.122283 16090 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111) [suppressed 5 similar messages]
17:11:17.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:17.124748 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647): Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:17.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:17.125355 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fbed83708289438c9a983ff8579912be (127.15.105.61:41549): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:17.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:17.125635 16090 leader_election.cc:302] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bb534d8de834a17b9d07567f42e6336; no voters: bd0d44b5c828438080d5a10cae87d03c, fbed83708289438c9a983ff8579912be
17:11:17.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:17.126124 16317 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
17:11:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:17.447943 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:17.448211 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:18.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:18.451087 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:18.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:18.451404 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:19.103 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:19.104 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:19.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:19.106 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:19.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:19.108 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:19.108 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:19.108 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:19.108 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:19.454561 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:19.454910 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:20.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:20 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497080, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:20.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:20.117326 16083 init.cc:295] Successfully reacquired a new kerberos TGT
17:11:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:20.450978 16067 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111),
17:11:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:20.451450 16067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:20.457798 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:20.458029 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:21.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:21.014647 16328 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:11:21.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:21.015110 16328 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:21.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:21.016937 16328 leader_election.cc:288] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647), fbed83708289438c9a983ff8579912be (127.15.105.61:41549)
17:11:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:21.021497 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647): Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:21.022184 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fbed83708289438c9a983ff8579912be (127.15.105.61:41549): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:21.022553 16090 leader_election.cc:302] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bb534d8de834a17b9d07567f42e6336; no voters: bd0d44b5c828438080d5a10cae87d03c, fbed83708289438c9a983ff8579912be
17:11:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:21.023175 16328 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
17:11:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:21.461088 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:21.461388 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:22.464362 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:22.464610 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:23.043 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:23.046 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:23.046 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:23.046 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:23.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:23.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:23.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:23.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:23.049 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:23.467504 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:23.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:23.467797 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:24.003 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:24.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:24.004 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:24.005 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:24.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:24.006 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:24.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:24.007 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:24.007 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:24.470536 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:24.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:24.470801 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:24.863 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:24.865 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:24.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:24.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:24.867 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:24.868 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:24.868 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:24.868 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:24.868 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:25.454092 16067 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111),
17:11:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:25.454452 16067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:25.473790 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:25.474046 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:26.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:26.476823 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:26.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:26.477090 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:27.479986 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:27.480252 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:27.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:27.878664 15625 debug-util.cc:398] Leaking SignalData structure 0x7b08000b05c0 after lost signal to thread 15500
17:11:27.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:27.879767 15625 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6380 after lost signal to thread 15628
17:11:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:28.100414 15765 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd480 after lost signal to thread 15640
17:11:28.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:28.101416 15765 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0300 after lost signal to thread 15768
17:11:28.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:28.234066 16339 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:11:28.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:28.234915 16339 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:28.237180 16339 leader_election.cc:288] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647), fbed83708289438c9a983ff8579912be (127.15.105.61:41549)
17:11:28.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:28.241403 16090 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111) [suppressed 3 similar messages]
17:11:28.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:28.246142 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647): Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:28.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:28.246757 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fbed83708289438c9a983ff8579912be (127.15.105.61:41549): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:28.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:28.247155 16090 leader_election.cc:302] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bb534d8de834a17b9d07567f42e6336; no voters: bd0d44b5c828438080d5a10cae87d03c, fbed83708289438c9a983ff8579912be
17:11:28.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:28.247912 16339 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
17:11:28.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:28.484555 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:28.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:28.489858 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:28.724 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:28.724 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:28.726 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:28.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:28.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:28.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:28.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:28.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:28.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:29.407667 15206 debug-util.cc:398] Leaking SignalData structure 0x7b0800083400 after lost signal to thread 15141
17:11:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:29.408782 15206 debug-util.cc:398] Leaking SignalData structure 0x7b0800083680 after lost signal to thread 15209
17:11:29.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:29.496753 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:29.497766 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:30.103 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:30.104 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:30.106 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:30.106 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:30.107 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:30.108 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:30.108 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:30.108 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:30.108 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:30 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497090, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:30.119657 16083 init.cc:295] Successfully reacquired a new kerberos TGT
17:11:30.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:30.457727 16067 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111),
17:11:30.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:30.458153 16067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:30.500967 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:30.501219 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:30.782 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:30.783 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:30.787 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:30.787 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:30.787 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:30.788 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:30.788 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:30.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:30.789 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:31.503928 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:31.504189 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:32.223 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:32.224 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:32.224 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:32.224 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:32.224 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:32.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:32.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:32.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:32.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:32.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:32.506836 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:32.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:32.507088 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:33.509779 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:33.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:33.510053 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:34.513002 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:34.513278 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:34.643 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:34.643 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:34.644 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:34.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:34.644 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:34.644 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:34.644 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:34.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:34.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:35.461241 16067 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111),
17:11:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:35.461726 16067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:35.516067 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:35.516337 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:36.103 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:36.104 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:36.104 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:36.104 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:36.104 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:36.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:36.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:36.106 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:36.106 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:36.519014 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:36.519281 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:37.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:37.522145 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:37.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:37.522403 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:38.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:38.525252 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:38.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:38.525530 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:39.423 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:39.424 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:39.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:39.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:39.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:39.426 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:39.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:39.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:39.429 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:39.528993 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:39.529328 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:39.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:39.946637 16358 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
17:11:39.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:39.947165 16358 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd0d44b5c828438080d5a10cae87d03c" member_type: VOTER last_known_addr { host: "127.15.105.62" port: 45647 } } peers { permanent_uuid: "fbed83708289438c9a983ff8579912be" member_type: VOTER last_known_addr { host: "127.15.105.61" port: 41549 } } peers { permanent_uuid: "4bb534d8de834a17b9d07567f42e6336" member_type: VOTER last_known_addr { host: "127.15.105.60" port: 40151 } }
17:11:39.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:39.948964 16358 leader_election.cc:288] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647), fbed83708289438c9a983ff8579912be (127.15.105.61:41549)
17:11:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:39.951535 16090 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111) [suppressed 1 similar messages]
17:11:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:39.954761 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd0d44b5c828438080d5a10cae87d03c (127.15.105.62:45647): Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111)
17:11:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:39.955485 16090 leader_election.cc:334] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fbed83708289438c9a983ff8579912be (127.15.105.61:41549): Network error: Client connection negotiation failed: client connection to 127.15.105.61:41549: connect: Connection refused (error 111)
17:11:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:39.955765 16090 leader_election.cc:302] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bb534d8de834a17b9d07567f42e6336; no voters: bd0d44b5c828438080d5a10cae87d03c, fbed83708289438c9a983ff8579912be
17:11:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:39.956336 16358 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
17:11:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 30 17:11:40 dist-test-slave-2p34 krb5kdc[15792](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714497100, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.105.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
17:11:40.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:40.122320 16083 init.cc:295] Successfully reacquired a new kerberos TGT
17:11:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:40.465023 16067 master_runner.cc:186] Error getting master registration for 127.15.105.62:45647: Network error: Client connection negotiation failed: client connection to 127.15.105.62:45647: connect: Connection refused (error 111),
17:11:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:40.465551 16067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
17:11:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:40.532881 16154 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4bb534d8de834a17b9d07567f42e6336: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
17:11:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:40.533246 16154 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
17:11:41.302 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:41.303 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:41.303 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:41.304 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:41.304 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:41.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:41.304 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:41.304 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:41.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:41.362 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.62:45647(127.15.105.62:45647)] connecting to peer
17:11:41.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.15.105.61:41549(127.15.105.61:41549)] connecting to peer
17:11:41.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:41.364 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.105.62:45647(127.15.105.62:45647)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:41.364 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.62:45647: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647
17:11:41.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:41.364 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.61:41549(127.15.105.61:41549)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:41.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.105.61:41549: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
17:11:41.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549; will retry
17:11:41.370 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27425), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [165ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [215ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [454ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [457ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [484ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [488ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [489ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [498ms] refreshing cache from master, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [503ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [504ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [504ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [505ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [518ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [524ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [526ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [539ms] refreshing cache from master, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [541ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [543ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [545ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [578ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [582ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [585ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [642ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [644ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [645ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [661ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [662ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [663ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [664ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [702ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [704ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [707ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1058ms] refreshing cache from master, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1064ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1718ms] refreshing cache from master, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1721ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1723ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1723ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1724ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1878ms] refreshing cache from master, [1878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1882ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [2378ms] refreshing cache from master, [2378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [2379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [2380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [2382ms] too many traces: truncated at 100 traces, deferred=Deferred@219528654(state=PENDING, result=null, callback=(continuation of Deferred@600835711 after retry RPC after error@1226080044) -> (continuation of Deferred@1163807714 after retry RPC after error@1703245130) -> (continuation of Deferred@1187341435 after retry RPC after error@742559376) -> (continuation of Deferred@469025185 after retry RPC after error@1435879600) -> (continuation of Deferred@949917893 after retry RPC after error@1095496) -> (continuation of Deferred@1028949794 after retry RPC after error@1182730352) -> (continuation of Deferred@174844106 after retry RPC after error@256160824) -> (continuation of Deferred@372268901 after retry RPC after error@682817883) -> (continuation of Deferred@2110417215 after retry RPC after error@1468397514) -> (continuation of Deferred@1129992488 after retry RPC after error@2090668089) -> (continuation of Deferred@1378880772 after retry RPC after error@720608047) -> (continuation of Deferred@993086536 after retry RPC after error@800694223) -> (continuation of Deferred@2100555537 after retry RPC after error@1261386107) -> (continuation of Deferred@2051899226 after retry RPC after error@434749499) -> (continuation of Deferred@2076241776 after retry RPC after error@1234625938) -> (continuation of Deferred@1487987797 after retry RPC after error@252649312) -> (continuation of Deferred@1942538910 after retry RPC after error@218893056) -> (continuation of Deferred@1380046008 after retry RPC after error@633480591) -> (continuation of Deferred@275486611 after retry RPC after error@2102581909) -> (continuation of Deferred@560608659 after retry RPC after error@355500652) -> (continuation of Deferred@1496008790 after retry RPC after error@851883924) -> (continuation of Deferred@1114692338 after retry RPC after error@1396839218) -> (continuation of Deferred@1344833780 after retry RPC after error@1465249650) -> (continuation of Deferred@322308128 after retry RPC after error@1353597687) -> (continuation of Deferred@1154817854 after retry RPC after error@1006851753), errback=(continuation of Deferred@600835711 after retry RPC after error@1226080044) -> (continuation of Deferred@1163807714 after retry RPC after error@1703245130) -> (continuation of Deferred@1187341435 after retry RPC after error@742559376) -> (continuation of Deferred@469025185 after retry RPC after error@1435879600) -> (continuation of Deferred@949917893 after retry RPC after error@1095496) -> (continuation of Deferred@1028949794 after retry RPC after error@1182730352) -> (continuation of Deferred@174844106 after retry RPC after error@256160824) -> (continuation of Deferred@372268901 after retry RPC after error@682817883) -> (continuation of Deferred@2110417215 after retry RPC after error@1468397514) -> (continuation of Deferred@1129992488 after retry RPC after error@2090668089) -> (continuation of Deferred@1378880772 after retry RPC after error@720608047) -> (continuation of Deferred@993086536 after retry RPC after error@800694223) -> (continuation of Deferred@2100555537 after retry RPC after error@1261386107) -> (continuation of Deferred@2051899226 after retry RPC after error@434749499) -> (continuation of Deferred@2076241776 after retry RPC after error@1234625938) -> (continuation of Deferred@1487987797 after retry RPC after error@252649312) -> (continuation of Deferred@1942538910 after retry RPC after error@218893056) -> (continuation of Deferred@1380046008 after retry RPC after error@633480591) -> (continuation of Deferred@275486611 after retry RPC after error@2102581909) -> (continuation of Deferred@560608659 after retry RPC after error@355500652) -> (continuation of Deferred@1496008790 after retry RPC after error@851883924) -> (continuation of Deferred@1114692338 after retry RPC after error@1396839218) -> (continuation of Deferred@1344833780 after retry RPC after error@1465249650) -> (continuation of Deferred@322308128 after retry RPC after error@1353597687) -> (continuation of Deferred@1154817854 after retry RPC after error@1006851753)))
17:11:41.375 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.105.60:40151(127.15.105.60:40151)] disconnecting while in state READY
17:11:41.377 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.105.61:41549(127.15.105.61:41549)] disconnecting while in state TERMINATED
17:11:41.377 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.15.105.62:45647(127.15.105.62:45647)] disconnecting while in state TERMINATED
17:11:41.381 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
17:11:41.382 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.105.60:40151(127.15.105.60:40151)] handling channelInactive
17:11:41.383 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.105.60:40151(127.15.105.60:40151)] cleaning up while in state READY due to: connection closed
17:11:41.388 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
17:11:41.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:41.390237 15780 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 16067
17:11:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T17:11:42Z chronyd exiting
17:11:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:42.531144 15484 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcfa0 after lost signal to thread 15355
17:11:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240430 17:11:42.532150 15484 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7800 after lost signal to thread 15488
17:11:43.527 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
17:11:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:43.528944 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15336
17:11:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:43.550496 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15496
17:11:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:43.577646 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15633
17:11:43.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:43.604004 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15067
17:11:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:43.634359 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15137
17:11:43.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240430 17:11:43.664350 15055 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNB5BOS/build/tsan/bin/kudu with pid 15212
17:11:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-30T17:11:44Z chronyd exiting
17:11:45.771 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27425), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [165ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [215ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [454ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [457ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [484ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [488ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [489ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [498ms] refreshing cache from master, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [503ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [504ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [504ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [505ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [518ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [524ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [526ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [539ms] refreshing cache from master, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [541ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [543ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [545ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [578ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [582ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [585ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [642ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [644ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [645ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [661ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [662ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [663ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [664ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [702ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [704ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [707ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1058ms] refreshing cache from master, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1064ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1718ms] refreshing cache from master, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1721ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1723ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1723ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1724ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1878ms] refreshing cache from master, [1878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1882ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [2378ms] refreshing cache from master, [2378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [2379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [2380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [2382ms] too many traces: truncated at 100 traces, deferred=Deferred@219528654(state=PENDING, result=null, callback=(continuation of Deferred@600835711 after retry RPC after error@1226080044) -> (continuation of Deferred@1163807714 after retry RPC after error@1703245130) -> (continuation of Deferred@1187341435 after retry RPC after error@742559376) -> (continuation of Deferred@469025185 after retry RPC after error@1435879600) -> (continuation of Deferred@949917893 after retry RPC after error@1095496) -> (continuation of Deferred@1028949794 after retry RPC after error@1182730352) -> (continuation of Deferred@174844106 after retry RPC after error@256160824) -> (continuation of Deferred@372268901 after retry RPC after error@682817883) -> (continuation of Deferred@2110417215 after retry RPC after error@1468397514) -> (continuation of Deferred@1129992488 after retry RPC after error@2090668089) -> (continuation of Deferred@1378880772 after retry RPC after error@720608047) -> (continuation of Deferred@993086536 after retry RPC after error@800694223) -> (continuation of Deferred@2100555537 after retry RPC after error@1261386107) -> (continuation of Deferred@2051899226 after retry RPC after error@434749499) -> (continuation of Deferred@2076241776 after retry RPC after error@1234625938) -> (continuation of Deferred@1487987797 after retry RPC after error@252649312) -> (continuation of Deferred@1942538910 after retry RPC after error@218893056) -> (continuation of Deferred@1380046008 after retry RPC after error@633480591) -> (continuation of Deferred@275486611 after retry RPC after error@2102581909) -> (continuation of Deferred@560608659 after retry RPC after error@355500652) -> (continuation of Deferred@1496008790 after retry RPC after error@851883924) -> (continuation of Deferred@1114692338 after retry RPC after error@1396839218) -> (continuation of Deferred@1344833780 after retry RPC after error@1465249650) -> (continuation of Deferred@322308128 after retry RPC after error@1353597687) -> (continuation of Deferred@1154817854 after retry RPC after error@1006851753), errback=(continuation of Deferred@600835711 after retry RPC after error@1226080044) -> (continuation of Deferred@1163807714 after retry RPC after error@1703245130) -> (continuation of Deferred@1187341435 after retry RPC after error@742559376) -> (continuation of Deferred@469025185 after retry RPC after error@1435879600) -> (continuation of Deferred@949917893 after retry RPC after error@1095496) -> (continuation of Deferred@1028949794 after retry RPC after error@1182730352) -> (continuation of Deferred@174844106 after retry RPC after error@256160824) -> (continuation of Deferred@372268901 after retry RPC after error@682817883) -> (continuation of Deferred@2110417215 after retry RPC after error@1468397514) -> (continuation of Deferred@1129992488 after retry RPC after error@2090668089) -> (continuation of Deferred@1378880772 after retry RPC after error@720608047) -> (continuation of Deferred@993086536 after retry RPC after error@800694223) -> (continuation of Deferred@2100555537 after retry RPC after error@1261386107) -> (continuation of Deferred@2051899226 after retry RPC after error@434749499) -> (continuation of Deferred@2076241776 after retry RPC after error@1234625938) -> (continuation of Deferred@1487987797 after retry RPC after error@252649312) -> (continuation of Deferred@1942538910 after retry RPC after error@218893056) -> (continuation of Deferred@1380046008 after retry RPC after error@633480591) -> (continuation of Deferred@275486611 after retry RPC after error@2102581909) -> (continuation of Deferred@560608659 after retry RPC after error@355500652) -> (continuation of Deferred@1496008790 after retry RPC after error@851883924) -> (continuation of Deferred@1114692338 after retry RPC after error@1396839218) -> (continuation of Deferred@1344833780 after retry RPC after error@1465249650) -> (continuation of Deferred@322308128 after retry RPC after error@1353597687) -> (continuation of Deferred@1154817854 after retry RPC after error@1006851753)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.105.61:41549
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=27425), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [165ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [215ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [454ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [457ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [478ms] refreshing cache from master, [479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [484ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [485ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [486ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [488ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [489ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [498ms] refreshing cache from master, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [500ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [503ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [504ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [504ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [505ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [518ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [522ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [524ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [526ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [539ms] refreshing cache from master, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [540ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [541ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [543ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [545ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [578ms] refreshing cache from master, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [582ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [583ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [585ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [638ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [642ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [644ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [645ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [661ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [662ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [663ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [664ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [701ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [702ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [704ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [707ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1058ms] refreshing cache from master, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1063ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1064ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1718ms] refreshing cache from master, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1721ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1723ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1723ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1724ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1878ms] refreshing cache from master, [1878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [1878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.62:45647: Network error: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.60:40151: OK, [1882ms] Sub RPC ConnectToMaster: received response from server master-127.15.105.61:41549: Network error: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [1882ms] delaying RPC due to: Service unavailable: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549, [2378ms] refreshing cache from master, [2378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.60:40151, [2379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.62:45647, [2380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.105.61:41549, [2382ms] too many traces: truncated at 100 traces, deferred=Deferred@219528654(state=PENDING, result=null, callback=(continuation of Deferred@600835711 after retry RPC after error@1226080044) -> (continuation of Deferred@1163807714 after retry RPC after error@1703245130) -> (continuation of Deferred@1187341435 after retry RPC after error@742559376) -> (continuation of Deferred@469025185 after retry RPC after error@1435879600) -> (continuation of Deferred@949917893 after retry RPC after error@1095496) -> (continuation of Deferred@1028949794 after retry RPC after error@1182730352) -> (continuation of Deferred@174844106 after retry RPC after error@256160824) -> (continuation of Deferred@372268901 after retry RPC after error@682817883) -> (continuation of Deferred@2110417215 after retry RPC after error@1468397514) -> (continuation of Deferred@1129992488 after retry RPC after error@2090668089) -> (continuation of Deferred@1378880772 after retry RPC after error@720608047) -> (continuation of Deferred@993086536 after retry RPC after error@800694223) -> (continuation of Deferred@2100555537 after retry RPC after error@1261386107) -> (continuation of Deferred@2051899226 after retry RPC after error@434749499) -> (continuation of Deferred@2076241776 after retry RPC after error@1234625938) -> (continuation of Deferred@1487987797 after retry RPC after error@252649312) -> (continuation of Deferred@1942538910 after retry RPC after error@218893056) -> (continuation of Deferred@1380046008 after retry RPC after error@633480591) -> (continuation of Deferred@275486611 after retry RPC after error@2102581909) -> (continuation of Deferred@560608659 after retry RPC after error@355500652) -> (continuation of Deferred@1496008790 after retry RPC after error@851883924) -> (continuation of Deferred@1114692338 after retry RPC after error@1396839218) -> (continuation of Deferred@1344833780 after retry RPC after error@1465249650) -> (continuation of Deferred@322308128 after retry RPC after error@1353597687) -> (continuation of Deferred@1154817854 after retry RPC after error@1006851753), errback=(continuation of Deferred@600835711 after retry RPC after error@1226080044) -> (continuation of Deferred@1163807714 after retry RPC after error@1703245130) -> (continuation of Deferred@1187341435 after retry RPC after error@742559376) -> (continuation of Deferred@469025185 after retry RPC after error@1435879600) -> (continuation of Deferred@949917893 after retry RPC after error@1095496) -> (continuation of Deferred@1028949794 after retry RPC after error@1182730352) -> (continuation of Deferred@174844106 after retry RPC after error@256160824) -> (continuation of Deferred@372268901 after retry RPC after error@682817883) -> (continuation of Deferred@2110417215 after retry RPC after error@1468397514) -> (continuation of Deferred@1129992488 after retry RPC after error@2090668089) -> (continuation of Deferred@1378880772 after retry RPC after error@720608047) -> (continuation of Deferred@993086536 after retry RPC after error@800694223) -> (continuation of Deferred@2100555537 after retry RPC after error@1261386107) -> (continuation of Deferred@2051899226 after retry RPC after error@434749499) -> (continuation of Deferred@2076241776 after retry RPC after error@1234625938) -> (continuation of Deferred@1487987797 after retry RPC after error@252649312) -> (continuation of Deferred@1942538910 after retry RPC after error@218893056) -> (continuation of Deferred@1380046008 after retry RPC after error@633480591) -> (continuation of Deferred@275486611 after retry RPC after error@2102581909) -> (continuation of Deferred@560608659 after retry RPC after error@355500652) -> (continuation of Deferred@1496008790 after retry RPC after error@851883924) -> (continuation of Deferred@1114692338 after retry RPC after error@1396839218) -> (continuation of Deferred@1344833780 after retry RPC after error@1465249650) -> (continuation of Deferred@322308128 after retry RPC after error@1353597687) -> (continuation of Deferred@1154817854 after retry RPC after error@1006851753)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.15.105.60:40151,127.15.105.62:45647,127.15.105.61:41549) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.62:45647(127.15.105.62:45647): Connection refused: /127.15.105.62:45647,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.15.105.61:41549(127.15.105.61:41549): Connection refused: /127.15.105.61:41549
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.15.105.61:41549
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)