08:16:27.431 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:16:27.443 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskiFzj_p/test-tmp
08:16:27.446 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:16:27.449 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskiFzj_p/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:16:27.672 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:16:27.832 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:16:27.833 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:16:27.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:27.838166 6677 mini_oidc.cc:152] Starting JWKS server
08:16:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:27.841652 6677 webserver.cc:460] Webserver started at http://127.0.0.1:41423/ using document root <none> and password file <none>
08:16:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:27.842101 6677 mini_oidc.cc:189] Starting OIDC Discovery server
08:16:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:27.844244 6677 webserver.cc:460] Webserver started at http://127.0.0.1:39563/ using document root <none> and password file <none>
08:16:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T08:16:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:16:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T08:16:27Z Disabled control of system clock
08:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:27.908607 6677 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/wal
08:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data
08:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/logs
08:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data/info.pb
08:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:27.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.126:41635
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.126
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:27.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41423/jwks.json with env {}
08:16:28.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.213071 6689 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.213640 6689 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.214402 6689 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:28.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.248294 6689 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.248601 6689 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.248795 6689 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.249014 6689 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.249301 6689 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.249529 6689 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41423/jwks.json
08:16:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:28.285471 6689 master_runner.cc:386] Master server non-default flags:
08:16:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data
08:16:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/wal
08:16:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.126:41635
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41423/jwks.json
08:16:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data/info.pb
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.126
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/logs
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:28.286759 6689 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:28.288372 6689 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:28.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.305984 6695 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:29.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.306525 6696 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:29.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:28.307821 6689 server_base.cc:1034] running on GCE node
08:16:29.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:28.307281 6698 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.425141 6689 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:29.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.428310 6689 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.429765 6689 hybrid_clock.cc:648] HybridClock initialized: now 1714724189429721 us; error 60 us; skew 500 ppm
08:16:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.430586 6689 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:29.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.437521 6689 webserver.cc:460] Webserver started at http://127.6.133.126:35695/ using document root <none> and password file <none>
08:16:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.438449 6689 fs_manager.cc:359] Metadata directory not provided
08:16:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.438688 6689 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:29.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.439110 6689 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.444152 6689 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data/instance:
08:16:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26ab38c2bd71499e80362a63502ff1ce"
08:16:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:29 on dist-test-slave-zl2t"
08:16:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.445559 6689 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/wal/instance:
08:16:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26ab38c2bd71499e80362a63502ff1ce"
08:16:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:29 on dist-test-slave-zl2t"
08:16:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.453768 6689 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
08:16:29.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.459554 6705 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:29.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.460690 6689 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:16:29.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.460990 6689 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/wal
08:16:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26ab38c2bd71499e80362a63502ff1ce"
08:16:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:29 on dist-test-slave-zl2t"
08:16:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.461308 6689 fs_report.cc:383] FS layout report
08:16:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/wal
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/wal
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data/data
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:29.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.528980 6689 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.530449 6689 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:29.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.530928 6689 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:29.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.600683 6689 rpc_server.cc:307] RPC server started. Bound to: 127.6.133.126:41635
08:16:29.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.600775 6756 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.133.126:41635 every 8 connection(s)
08:16:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.603430 6689 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/data/info.pb
08:16:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.610245 6757 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.612335 6677 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 6689
08:16:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.612709 6677 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-0/wal/instance
08:16:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.616039 6677 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/wal
08:16:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/logs
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data/info.pb
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.125:43571
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.125
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41423/jwks.json with env {}
08:16:29.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.624156 6757 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } has no permanent_uuid. Determining permanent_uuid...
08:16:29.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.652688 6757 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } has no permanent_uuid. Determining permanent_uuid...
08:16:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.655082 6707 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.659343 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.125:43571: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:29.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.708822 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } attempt: 1
08:16:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.713446 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.125:43571: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.781831 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } attempt: 2
08:16:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.786108 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.125:43571: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.877527 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } attempt: 3
08:16:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.881753 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.125:43571: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.925446 6759 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.926168 6759 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.926841 6759 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:29.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.960237 6759 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.960615 6759 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.960951 6759 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.961311 6759 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:29.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.961652 6759 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:29.961890 6759 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41423/jwks.json
08:16:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.996625 6759 master_runner.cc:386] Master server non-default flags:
08:16:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data
08:16:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/wal
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.125:43571
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41423/jwks.json
08:16:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data/info.pb
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.125
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/logs
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:30.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:30.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:30.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:30.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:30.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:30.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:30.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:29.998168 6759 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:30.000468 6759 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:30.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:30.017150 6768 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:30.025146 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } attempt: 4
08:16:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:30.028978 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.125:43571: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:30.328460 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } attempt: 5
08:16:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:30.332202 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.125:43571: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:30.879930 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } attempt: 6
08:16:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:30.893895 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.125:43571: Network error: Client connection negotiation failed: client connection to 127.6.133.125:43571: connect: Connection refused (error 111)
08:16:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:30.018177 6771 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:30.019697 6759 server_base.cc:1034] running on GCE node
08:16:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:30.017943 6769 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:31.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.136478 6759 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:31.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.139845 6759 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:31.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.141309 6759 hybrid_clock.cc:648] HybridClock initialized: now 1714724191141266 us; error 62 us; skew 500 ppm
08:16:31.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.143157 6759 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:31.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.154167 6759 webserver.cc:460] Webserver started at http://127.6.133.125:40113/ using document root <none> and password file <none>
08:16:31.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.155205 6759 fs_manager.cc:359] Metadata directory not provided
08:16:31.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.155423 6759 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:31.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.155825 6759 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.161002 6759 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data/instance:
08:16:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5fb9b2723697436699969d39f23498ad"
08:16:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:31 on dist-test-slave-zl2t"
08:16:31.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.162190 6759 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/wal/instance:
08:16:31.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5fb9b2723697436699969d39f23498ad"
08:16:31.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:31 on dist-test-slave-zl2t"
08:16:31.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.169402 6759 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
08:16:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.175067 6780 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.176225 6759 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:16:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.176558 6759 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/wal
08:16:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5fb9b2723697436699969d39f23498ad"
08:16:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:31 on dist-test-slave-zl2t"
08:16:31.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.176867 6759 fs_report.cc:383] FS layout report
08:16:31.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:31.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/wal
08:16:31.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/wal
08:16:31.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data/data
08:16:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.229058 6759 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.230499 6759 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:31.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.230939 6759 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.305106 6759 rpc_server.cc:307] RPC server started. Bound to: 127.6.133.125:43571
08:16:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.305218 6831 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.133.125:43571 every 8 connection(s)
08:16:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.307942 6759 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/data/info.pb
08:16:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.313268 6832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.314365 6677 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 6759
08:16:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.314849 6677 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-1/wal/instance
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.317932 6677 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/wal
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/logs
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data/info.pb
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.124:39343
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.124
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:31.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41423/jwks.json with env {}
08:16:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.328466 6832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } has no permanent_uuid. Determining permanent_uuid...
08:16:31.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.354926 6832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } has no permanent_uuid. Determining permanent_uuid...
08:16:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.369632 6832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } has no permanent_uuid. Determining permanent_uuid...
08:16:31.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.372309 6783 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:31.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.376479 6832 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.426020 6832 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 1
08:16:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.430413 6832 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.498854 6832 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 2
08:16:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.503616 6832 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:31.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.595110 6832 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 3
08:16:31.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.599788 6832 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:31.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.641391 6834 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:31.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.642086 6834 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:31.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.642828 6834 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.674185 6834 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.674552 6834 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.674901 6834 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.675256 6834 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.675575 6834 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:31.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.675825 6834 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41423/jwks.json
08:16:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.710960 6834 master_runner.cc:386] Master server non-default flags:
08:16:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/wal
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.124:39343
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41423/jwks.json
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data/info.pb
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.124
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/logs
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:31.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.712479 6834 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:31.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.714828 6834 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.732774 6844 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.743319 6832 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 4
08:16:31.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.747953 6832 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.954576 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } attempt: 7
08:16:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.973160 6757 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } has no permanent_uuid. Determining permanent_uuid...
08:16:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.979543 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.038257 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 1
08:16:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:32.042685 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.047430 6832 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 5
08:16:32.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:32.053642 6832 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.124392 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 2
08:16:32.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:32.128719 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.214349 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 3
08:16:32.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:32.219450 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.359834 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 4
08:16:32.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:32.364948 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.601279 6832 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 6
08:16:32.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:32.610085 6832 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.648404 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 5
08:16:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:32.652863 6757 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.6.133.124:39343: Network error: Client connection negotiation failed: client connection to 127.6.133.124:39343: connect: Connection refused (error 111)
08:16:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.733472 6845 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:31.734601 6834 server_base.cc:1034] running on GCE node
08:16:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:31.733978 6847 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.865372 6834 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:32.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.868171 6834 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:32.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.869544 6834 hybrid_clock.cc:648] HybridClock initialized: now 1714724192869496 us; error 55 us; skew 500 ppm
08:16:32.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.870463 6834 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.876758 6834 webserver.cc:460] Webserver started at http://127.6.133.124:44635/ using document root <none> and password file <none>
08:16:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.877758 6834 fs_manager.cc:359] Metadata directory not provided
08:16:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.877981 6834 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.878545 6834 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.884076 6834 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data/instance:
08:16:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "058971fab38243e9b8f1f1b781bc4fc1"
08:16:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:32 on dist-test-slave-zl2t"
08:16:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.885427 6834 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/wal/instance:
08:16:32.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "058971fab38243e9b8f1f1b781bc4fc1"
08:16:32.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:32 on dist-test-slave-zl2t"
08:16:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.895524 6834 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.000s
08:16:32.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.901494 6859 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:32.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.902597 6834 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:16:32.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.903019 6834 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/wal
08:16:32.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "058971fab38243e9b8f1f1b781bc4fc1"
08:16:32.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:32 on dist-test-slave-zl2t"
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.903416 6834 fs_report.cc:383] FS layout report
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/wal
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/wal
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data/data
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:32.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:32.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:32.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.965899 6834 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:32.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.967478 6834 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:32.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:32.967978 6834 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.035302 6834 rpc_server.cc:307] RPC server started. Bound to: 127.6.133.124:39343
08:16:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.035401 6910 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.133.124:39343 every 8 connection(s)
08:16:33.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.038053 6834 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/data/info.pb
08:16:33.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.043502 6677 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 6834
08:16:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.043740 6911 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.043941 6677 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/master-2/wal/instance
08:16:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.062474 6911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } has no permanent_uuid. Determining permanent_uuid...
08:16:33.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.096762 6911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } has no permanent_uuid. Determining permanent_uuid...
08:16:33.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.105847 6911 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } has no permanent_uuid. Determining permanent_uuid...
08:16:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.121848 6911 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1: Bootstrap starting.
08:16:33.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.128399 6911 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1: Neither blocks nor log segments found. Creating new log.
08:16:33.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.130492 6911 log.cc:824] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1: Log is configured to *not* fsync() on all Append() calls
08:16:33.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.134982 6911 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1: No bootstrap required, opened a new log
08:16:33.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.161150 6911 raft_consensus.cc:348] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.161906 6911 raft_consensus.cc:374] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:16:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.162189 6911 raft_consensus.cc:729] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 058971fab38243e9b8f1f1b781bc4fc1, State: Initialized, Role: FOLLOWER
08:16:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.163130 6911 consensus_queue.cc:260] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.166219 6920 sys_catalog.cc:455] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.166990 6920 sys_catalog.cc:458] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: This master's current role is: FOLLOWER
08:16:33.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.168205 6911 sys_catalog.cc:564] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: configured and running, proceeding with master startup.
08:16:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:33.193646 6931 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:33.194440 6931 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.205304 6757 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 6
08:16:33.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:33.222952 6865 tablet.cc:2369] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:16:33.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.190078 6834 master_runner.cc:186] Error getting master registration for 127.6.133.126:41635: OK, instance_id { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" instance_seqno: 1714724189580740 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:16:33.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.224671 6834 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:33.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.224978 6757 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: Bootstrap starting.
08:16:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.233019 6757 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: Neither blocks nor log segments found. Creating new log.
08:16:33.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.235368 6757 log.cc:824] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: Log is configured to *not* fsync() on all Append() calls
08:16:33.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.240340 6757 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: No bootstrap required, opened a new log
08:16:33.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.262960 6757 raft_consensus.cc:348] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.263621 6757 raft_consensus.cc:374] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:16:33.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.263886 6757 raft_consensus.cc:729] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26ab38c2bd71499e80362a63502ff1ce, State: Initialized, Role: FOLLOWER
08:16:33.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.264779 6757 consensus_queue.cc:260] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.267765 6935 sys_catalog.cc:455] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.269534 6935 sys_catalog.cc:458] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [sys.catalog]: This master's current role is: FOLLOWER
08:16:33.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.270085 6757 sys_catalog.cc:564] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [sys.catalog]: configured and running, proceeding with master startup.
08:16:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.273427 6920 raft_consensus.cc:482] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:33.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.274919 6920 raft_consensus.cc:504] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.279076 6920 leader_election.cc:288] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 26ab38c2bd71499e80362a63502ff1ce (127.6.133.126:41635), 5fb9b2723697436699969d39f23498ad (127.6.133.125:43571)
08:16:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.279431 6732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "058971fab38243e9b8f1f1b781bc4fc1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26ab38c2bd71499e80362a63502ff1ce" is_pre_election: true
08:16:33.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.280402 6732 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 058971fab38243e9b8f1f1b781bc4fc1 in term 0.
08:16:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.282325 6860 leader_election.cc:302] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 058971fab38243e9b8f1f1b781bc4fc1, 26ab38c2bd71499e80362a63502ff1ce; no voters:
08:16:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.283228 6920 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:16:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.283560 6920 raft_consensus.cc:482] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:16:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.283865 6920 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 0 FOLLOWER]: Advancing to term 1
08:16:33.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:33.289737 6711 tablet.cc:2369] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:16:33.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.292096 6920 raft_consensus.cc:504] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.294967 6732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "058971fab38243e9b8f1f1b781bc4fc1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "26ab38c2bd71499e80362a63502ff1ce"
08:16:33.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.295554 6732 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [term 0 FOLLOWER]: Advancing to term 1
08:16:33.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.300030 6920 leader_election.cc:288] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [CANDIDATE]: Term 1 election: Requested vote from peers 26ab38c2bd71499e80362a63502ff1ce (127.6.133.126:41635), 5fb9b2723697436699969d39f23498ad (127.6.133.125:43571)
08:16:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.303510 6732 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 058971fab38243e9b8f1f1b781bc4fc1 in term 1.
08:16:33.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.304497 6860 leader_election.cc:302] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 058971fab38243e9b8f1f1b781bc4fc1, 26ab38c2bd71499e80362a63502ff1ce; no voters:
08:16:33.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.305722 6920 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 1 FOLLOWER]: Leader election won for term 1
08:16:33.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.310974 6920 raft_consensus.cc:686] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [term 1 LEADER]: Becoming Leader. State: Replica: 058971fab38243e9b8f1f1b781bc4fc1, State: Running, Role: LEADER
08:16:33.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.312229 6920 consensus_queue.cc:237] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.340217 6946 sys_catalog.cc:455] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 058971fab38243e9b8f1f1b781bc4fc1. Latest consensus state: current_term: 1 leader_uuid: "058971fab38243e9b8f1f1b781bc4fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.341274 6946 sys_catalog.cc:458] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: This master's current role is: LEADER
08:16:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.358664 6949 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
08:16:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:33.359297 6947 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:33.359640 6947 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:33.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.369086 6949 catalog_manager.cc:1484] Initializing Kudu cluster ID...
08:16:33.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.334247 6689 master_runner.cc:186] Error getting master registration for 127.6.133.125:43571: OK, instance_id { permanent_uuid: "5fb9b2723697436699969d39f23498ad" instance_seqno: 1714724191284231 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:16:33.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.383622 6689 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:33.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.396859 6732 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [term 1 FOLLOWER]: Refusing update from remote peer 058971fab38243e9b8f1f1b781bc4fc1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:16:33.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.398737 6920 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:16:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.437165 6935 sys_catalog.cc:455] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [sys.catalog]: SysCatalogTable state changed. Reason: New leader 058971fab38243e9b8f1f1b781bc4fc1. Latest consensus state: current_term: 1 leader_uuid: "058971fab38243e9b8f1f1b781bc4fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.437968 6935 sys_catalog.cc:458] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [sys.catalog]: This master's current role is: FOLLOWER
08:16:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.456890 6935 sys_catalog.cc:455] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "058971fab38243e9b8f1f1b781bc4fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.457690 6935 sys_catalog.cc:458] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce [sys.catalog]: This master's current role is: FOLLOWER
08:16:33.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.476070 6920 sys_catalog.cc:455] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "058971fab38243e9b8f1f1b781bc4fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.476701 6920 sys_catalog.cc:458] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: This master's current role is: LEADER
08:16:33.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.488605 6949 catalog_manager.cc:1347] Generated new cluster ID: 293152bc121e489592981631b62ffeb5
08:16:33.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.488926 6949 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
08:16:33.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.548254 6949 catalog_manager.cc:1370] Generated new certificate authority record
08:16:33.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.551565 6949 catalog_manager.cc:1504] Loading token signing keys...
08:16:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.574461 6949 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1: Generated new TSK 0
08:16:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.577690 6949 catalog_manager.cc:1514] Initializing in-progress tserver states...
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.601711 6677 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/wal
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/logs
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data/info.pb
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.65:0
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.133.65
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.65
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:33.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:16:33.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.670444 6832 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } attempt: 7
08:16:33.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.699959 6832 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad: Bootstrap starting.
08:16:33.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.731048 6832 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad: Neither blocks nor log segments found. Creating new log.
08:16:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.734126 6832 log.cc:824] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad: Log is configured to *not* fsync() on all Append() calls
08:16:33.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.750869 6832 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad: No bootstrap required, opened a new log
08:16:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.838774 6832 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.839686 6832 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:16:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.840037 6832 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5fb9b2723697436699969d39f23498ad, State: Initialized, Role: FOLLOWER
08:16:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.841048 6832 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } }
08:16:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.854535 6959 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.857060 6959 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [sys.catalog]: This master's current role is: FOLLOWER
08:16:33.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.865309 6832 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [sys.catalog]: configured and running, proceeding with master startup.
08:16:33.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.865501 6807 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "058971fab38243e9b8f1f1b781bc4fc1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5fb9b2723697436699969d39f23498ad" is_pre_election: true
08:16:33.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.866434 6807 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 058971fab38243e9b8f1f1b781bc4fc1 in term 0.
08:16:33.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.870249 6806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "058971fab38243e9b8f1f1b781bc4fc1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5fb9b2723697436699969d39f23498ad"
08:16:33.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.870904 6806 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [term 0 FOLLOWER]: Advancing to term 1
08:16:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.878912 6806 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 058971fab38243e9b8f1f1b781bc4fc1 in term 1.
08:16:33.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.880540 6805 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [term 1 FOLLOWER]: Refusing update from remote peer 058971fab38243e9b8f1f1b781bc4fc1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:16:33.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.889012 6920 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:16:33.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.941354 6959 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [sys.catalog]: SysCatalogTable state changed. Reason: New leader 058971fab38243e9b8f1f1b781bc4fc1. Latest consensus state: current_term: 1 leader_uuid: "058971fab38243e9b8f1f1b781bc4fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.942162 6959 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [sys.catalog]: This master's current role is: FOLLOWER
08:16:33.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.944160 6966 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "058971fab38243e9b8f1f1b781bc4fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:33.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:33.945986 6966 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad [sys.catalog]: This master's current role is: FOLLOWER
08:16:34.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.016026 6975 catalog_manager.cc:1259] Loaded cluster ID: 293152bc121e489592981631b62ffeb5
08:16:34.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.016659 6975 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad: loading cluster ID for follower catalog manager: success
08:16:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.023907 6975 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad: acquiring CA information for follower catalog manager: success
08:16:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.039062 6975 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5fb9b2723697436699969d39f23498ad: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:16:34.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.098269 6956 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:34.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.099292 6956 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.100271 6956 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.158460 6956 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.160218 6956 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.133.65
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.221838 6956 tablet_server_runner.cc:78] Tablet server non-default flags:
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/wal
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.65:0
08:16:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data/info.pb
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.65
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:34.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.133.65
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/logs
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:34.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:34.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.225829 6956 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.228178 6956 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.278309 6983 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.365864 6947 catalog_manager.cc:1259] Loaded cluster ID: 293152bc121e489592981631b62ffeb5
08:16:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.366506 6947 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: loading cluster ID for follower catalog manager: success
08:16:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.376886 6947 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: acquiring CA information for follower catalog manager: success
08:16:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.405504 6947 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 26ab38c2bd71499e80362a63502ff1ce: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:16:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.445591 6920 sys_catalog.cc:455] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "058971fab38243e9b8f1f1b781bc4fc1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26ab38c2bd71499e80362a63502ff1ce" member_type: VOTER last_known_addr { host: "127.6.133.126" port: 41635 } } peers { permanent_uuid: "5fb9b2723697436699969d39f23498ad" member_type: VOTER last_known_addr { host: "127.6.133.125" port: 43571 } } peers { permanent_uuid: "058971fab38243e9b8f1f1b781bc4fc1" member_type: VOTER last_known_addr { host: "127.6.133.124" port: 39343 } } }
08:16:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:34.447186 6920 sys_catalog.cc:458] T 00000000000000000000000000000000 P 058971fab38243e9b8f1f1b781bc4fc1 [sys.catalog]: This master's current role is: LEADER
08:16:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.291342 6986 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:35.677845 6984 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:16:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.678041 6956 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:16:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:34.282544 6982 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.690873 6956 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.701375 6956 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.703078 6956 hybrid_clock.cc:648] HybridClock initialized: now 1714724195703010 us; error 84 us; skew 500 ppm
08:16:35.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.704346 6956 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:35.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.734203 6956 webserver.cc:460] Webserver started at http://127.6.133.65:46207/ using document root <none> and password file <none>
08:16:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.735893 6956 fs_manager.cc:359] Metadata directory not provided
08:16:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.736397 6956 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.737121 6956 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.744585 6956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data/instance:
08:16:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9734ffad3514a19a1f7e541cfe44e25"
08:16:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:35 on dist-test-slave-zl2t"
08:16:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.747152 6956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/wal/instance:
08:16:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9734ffad3514a19a1f7e541cfe44e25"
08:16:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:35 on dist-test-slave-zl2t"
08:16:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.762962 6956 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.005s sys 0.008s
08:16:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.776634 6994 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.778326 6956 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.001s sys 0.007s
08:16:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.778976 6956 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/wal
08:16:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9734ffad3514a19a1f7e541cfe44e25"
08:16:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:35 on dist-test-slave-zl2t"
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.779762 6956 fs_report.cc:383] FS layout report
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/wal
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/wal
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data/data
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:35.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.875679 6956 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:35.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.878024 6956 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:35.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.878913 6956 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:35.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.888212 6956 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:16:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.900306 6956 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:16:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.900920 6956 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.001s
08:16:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.901412 6956 ts_tablet_manager.cc:590] Registered 0 tablets
08:16:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:35.901783 6956 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:16:36.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.715646 6956 rpc_server.cc:307] RPC server started. Bound to: 127.6.133.65:34145
08:16:36.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.715762 7106 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.133.65:34145 every 8 connection(s)
08:16:36.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.718659 6956 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/data/info.pb
08:16:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.724176 6677 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 6956
08:16:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.724761 6677 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-0/wal/instance
08:16:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.739338 6677 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/wal
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/logs
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data/info.pb
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.66:0
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.133.66
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.66
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:16:36.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.810290 7108 heartbeater.cc:347] Connected to a master server at 127.6.133.125:43571
08:16:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.810828 7108 heartbeater.cc:464] Registering TS with master...
08:16:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.812671 7108 heartbeater.cc:511] Master 127.6.133.125:43571 requested a full tablet report, sending...
08:16:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.814997 7107 heartbeater.cc:347] Connected to a master server at 127.6.133.124:39343
08:16:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.815384 7107 heartbeater.cc:464] Registering TS with master...
08:16:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.816270 7107 heartbeater.cc:511] Master 127.6.133.124:39343 requested a full tablet report, sending...
08:16:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.816455 6797 ts_manager.cc:194] Registered new tserver with Master: d9734ffad3514a19a1f7e541cfe44e25 (127.6.133.65:34145)
08:16:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.819214 6876 ts_manager.cc:194] Registered new tserver with Master: d9734ffad3514a19a1f7e541cfe44e25 (127.6.133.65:34145)
08:16:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.821851 6876 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.133.65:53009
08:16:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.822510 7111 heartbeater.cc:347] Connected to a master server at 127.6.133.126:41635
08:16:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.822875 7111 heartbeater.cc:464] Registering TS with master...
08:16:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.824025 7111 heartbeater.cc:511] Master 127.6.133.126:41635 requested a full tablet report, sending...
08:16:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:36.827697 6722 ts_manager.cc:194] Registered new tserver with Master: d9734ffad3514a19a1f7e541cfe44e25 (127.6.133.65:34145)
08:16:37.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.178917 7116 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:37.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.179482 7116 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:37.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.180056 7116 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.211796 7116 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.213024 7116 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.133.66
08:16:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:37.250196 7116 tablet_server_runner.cc:78] Tablet server non-default flags:
08:16:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/wal
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.66:0
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data/info.pb
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.66
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.133.66
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/logs
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:37.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:16:37.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:37.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:37.251734 7116 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:37.253834 7116 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.272548 7124 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:37.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:37.832005 7107 heartbeater.cc:503] Master 127.6.133.124:39343 was elected leader, sending a full tablet report...
08:16:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.273881 7126 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:37.275839 7116 server_base.cc:1034] running on GCE node
08:16:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:37.273229 7123 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.402400 7116 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.404671 7116 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.406049 7116 hybrid_clock.cc:648] HybridClock initialized: now 1714724198405991 us; error 62 us; skew 500 ppm
08:16:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.406888 7116 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:38.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.413667 7116 webserver.cc:460] Webserver started at http://127.6.133.66:46141/ using document root <none> and password file <none>
08:16:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.414577 7116 fs_manager.cc:359] Metadata directory not provided
08:16:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.414824 7116 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.415232 7116 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.420065 7116 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data/instance:
08:16:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e29e6546c9b4231bb683791b6063499"
08:16:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:38 on dist-test-slave-zl2t"
08:16:38.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.421111 7116 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/wal/instance:
08:16:38.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e29e6546c9b4231bb683791b6063499"
08:16:38.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:38 on dist-test-slave-zl2t"
08:16:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.428294 7116 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
08:16:38.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.434020 7135 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.435166 7116 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:16:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.435473 7116 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/wal
08:16:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e29e6546c9b4231bb683791b6063499"
08:16:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:38 on dist-test-slave-zl2t"
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.435832 7116 fs_report.cc:383] FS layout report
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/wal
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/wal
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data/data
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:38.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:38.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.499152 7116 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:38.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.500577 7116 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:38.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.501005 7116 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.503564 7116 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:16:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.507741 7116 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:16:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.507936 7116 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:16:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.508131 7116 ts_tablet_manager.cc:590] Registered 0 tablets
08:16:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.508266 7116 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:16:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.666203 7116 rpc_server.cc:307] RPC server started. Bound to: 127.6.133.66:44753
08:16:38.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.666282 7247 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.133.66:44753 every 8 connection(s)
08:16:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.668816 7116 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/data/info.pb
08:16:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.671032 6677 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7116
08:16:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.671641 6677 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-1/wal/instance
08:16:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.703285 6677 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/wal
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/logs
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data/info.pb
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.67:0
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.133.67
08:16:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.67
08:16:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:38.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:16:38.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.733922 7249 heartbeater.cc:347] Connected to a master server at 127.6.133.125:43571
08:16:38.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.734438 7249 heartbeater.cc:464] Registering TS with master...
08:16:38.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.736106 7249 heartbeater.cc:511] Master 127.6.133.125:43571 requested a full tablet report, sending...
08:16:38.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.736784 7250 heartbeater.cc:347] Connected to a master server at 127.6.133.126:41635
08:16:38.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.737130 7250 heartbeater.cc:464] Registering TS with master...
08:16:38.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.737617 7248 heartbeater.cc:347] Connected to a master server at 127.6.133.124:39343
08:16:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.737953 7248 heartbeater.cc:464] Registering TS with master...
08:16:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.738024 7250 heartbeater.cc:511] Master 127.6.133.126:41635 requested a full tablet report, sending...
08:16:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.739432 7248 heartbeater.cc:511] Master 127.6.133.124:39343 requested a full tablet report, sending...
08:16:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.739777 6797 ts_manager.cc:194] Registered new tserver with Master: 9e29e6546c9b4231bb683791b6063499 (127.6.133.66:44753)
08:16:38.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.741657 6876 ts_manager.cc:194] Registered new tserver with Master: 9e29e6546c9b4231bb683791b6063499 (127.6.133.66:44753)
08:16:38.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.742241 6722 ts_manager.cc:194] Registered new tserver with Master: 9e29e6546c9b4231bb683791b6063499 (127.6.133.66:44753)
08:16:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:38.743515 6876 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.133.66:38759
08:16:39.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.059031 7258 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:39.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.059576 7258 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:39.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.060156 7258 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:39.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.092104 7258 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.093051 7258 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.133.67
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:39.126782 7258 tablet_server_runner.cc:78] Tablet server non-default flags:
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.133.84:39165
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/wal
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.133.67:0
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data/info.pb
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.133.67
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.133.126:41635,127.6.133.125:43571,127.6.133.124:39343
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.133.67
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/logs
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:39.128300 7258 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:39.129972 7258 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.153079 7264 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:39.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:39.746778 7248 heartbeater.cc:503] Master 127.6.133.124:39343 was elected leader, sending a full tablet report...
08:16:40.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:40.008154 6907 debug-util.cc:398] Leaking SignalData structure 0x7b08000bca80 after lost signal to thread 6839
08:16:40.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:40.008821 6907 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8da0 after lost signal to thread 6910
08:16:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:39.155509 7258 server_base.cc:1034] running on GCE node
08:16:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.154218 7265 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:39.153352 7267 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.286417 7258 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:40.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.289489 7258 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:40.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.291174 7258 hybrid_clock.cc:648] HybridClock initialized: now 1714724200291113 us; error 57 us; skew 500 ppm
08:16:40.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.292307 7258 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:40.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.300483 7258 webserver.cc:460] Webserver started at http://127.6.133.67:46717/ using document root <none> and password file <none>
08:16:40.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.301745 7258 fs_manager.cc:359] Metadata directory not provided
08:16:40.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.302045 7258 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:40.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.302632 7258 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.310711 7258 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data/instance:
08:16:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "860744bfc4a4408999909fd7b724cea5"
08:16:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:40 on dist-test-slave-zl2t"
08:16:40.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.312502 7258 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/wal/instance:
08:16:40.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "860744bfc4a4408999909fd7b724cea5"
08:16:40.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:40 on dist-test-slave-zl2t"
08:16:40.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.322901 7258 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
08:16:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.331023 7277 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.332104 7258 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:16:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.332383 7258 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/wal
08:16:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "860744bfc4a4408999909fd7b724cea5"
08:16:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:40 on dist-test-slave-zl2t"
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.332705 7258 fs_report.cc:383] FS layout report
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/wal
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/wal
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data/data
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.398800 7258 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.400197 7258 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.400599 7258 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.403165 7258 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:16:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.407603 7258 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:16:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.407829 7258 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:16:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.408161 7258 ts_tablet_manager.cc:590] Registered 0 tablets
08:16:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.408339 7258 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:16:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.614651 7258 rpc_server.cc:307] RPC server started. Bound to: 127.6.133.67:36071
08:16:40.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.614755 7390 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.133.67:36071 every 8 connection(s)
08:16:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.618503 7258 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/data/info.pb
08:16:40.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.623203 6677 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7258
08:16:40.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.623685 6677 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster4885047953501690557/ts-2/wal/instance
08:16:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.675361 7391 heartbeater.cc:347] Connected to a master server at 127.6.133.124:39343
08:16:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.676020 7391 heartbeater.cc:464] Registering TS with master...
08:16:40.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.677628 7391 heartbeater.cc:511] Master 127.6.133.124:39343 requested a full tablet report, sending...
08:16:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.678818 7392 heartbeater.cc:347] Connected to a master server at 127.6.133.125:43571
08:16:40.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.679196 7392 heartbeater.cc:464] Registering TS with master...
08:16:40.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.680202 7392 heartbeater.cc:511] Master 127.6.133.125:43571 requested a full tablet report, sending...
08:16:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.681507 6876 ts_manager.cc:194] Registered new tserver with Master: 860744bfc4a4408999909fd7b724cea5 (127.6.133.67:36071)
08:16:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.683269 6876 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.133.67:50035
08:16:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.683630 6797 ts_manager.cc:194] Registered new tserver with Master: 860744bfc4a4408999909fd7b724cea5 (127.6.133.67:36071)
08:16:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.695164 7395 heartbeater.cc:347] Connected to a master server at 127.6.133.126:41635
08:16:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.695570 7395 heartbeater.cc:464] Registering TS with master...
08:16:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.696272 7395 heartbeater.cc:511] Master 127.6.133.126:41635 requested a full tablet report, sending...
08:16:40.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.703748 6722 ts_manager.cc:194] Registered new tserver with Master: 860744bfc4a4408999909fd7b724cea5 (127.6.133.67:36071)
08:16:40.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:40.715266 6677 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
08:16:40.726 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:16:40.727 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:16:40.738 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.133.126"
port: 41635
}
bound_http_address {
host: "127.6.133.126"
port: 35695
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.133.125"
port: 43571
}
bound_http_address {
host: "127.6.133.125"
port: 40113
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.133.124"
port: 39343
}
bound_http_address {
host: "127.6.133.124"
port: 44635
}
}
}
08:16:40.744 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:16:40.748 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.133.65"
port: 34145
}
bound_http_address {
host: "127.6.133.65"
port: 46207
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.133.66"
port: 44753
}
bound_http_address {
host: "127.6.133.66"
port: 46141
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.133.67"
port: 36071
}
bound_http_address {
host: "127.6.133.67"
port: 46717
}
}
}
08:16:40.749 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:16:41.005 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:16:41.171 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskiFzj_p/test-tmp
08:16:41.173 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:16:41.173 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskiFzj_p/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:16:41.178 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:16:41.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:41.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:16:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:41.598785 7402 mini_oidc.cc:152] Starting JWKS server
08:16:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:41.603127 7402 webserver.cc:460] Webserver started at http://127.0.0.1:34029/ using document root <none> and password file <none>
08:16:41.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:41.603717 7402 mini_oidc.cc:189] Starting OIDC Discovery server
08:16:41.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:41.607303 7402 webserver.cc:460] Webserver started at http://127.0.0.1:46477/ using document root <none> and password file <none>
08:16:41.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:41.688535 7391 heartbeater.cc:503] Master 127.6.133.124:39343 was elected leader, sending a full tablet report...
08:16:41.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:16:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/principal' for realm 'KRBTEST.COM',
08:16:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:16:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:41 dist-test-slave-zl2t krb5kdc[7415](info): setting up network...
08:16:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:16:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:41 dist-test-slave-zl2t krb5kdc[7415](info): set up 2 sockets
08:16:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:41 dist-test-slave-zl2t krb5kdc[7415](info): commencing operation
08:16:41.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:16:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:42.715222 7402 mini_kdc.cc:121] Time spent starting KDC: real 1.048s user 0.002s sys 0.021s
08:16:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:16:42.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:16:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:16:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:16:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:16:42.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:42.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:16:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:16:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/test-user.keytab.
08:16:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/test-user.keytab.
08:16:42.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:42 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724202, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:42.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:16:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T08:16:42Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:16:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T08:16:42Z Disabled control of system clock
08:16:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.58.190@KRBTEST.COM; defaulting to no policy
08:16:42.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:42.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.58.190@KRBTEST.COM" created.
08:16:42.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:42.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.58.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.190.keytab.
08:16:42.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.58.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.190.keytab.
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:42.913084 7402 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/logs
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/info.pb
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:42.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.190:45779
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.190
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.190.keytab
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.190
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:43.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.247792 7428 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:43.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.248538 7428 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.249253 7428 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:43.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.281985 7428 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:43.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.282258 7428 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:43.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.282708 7428 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:43.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.282992 7428 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:43.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.283198 7428 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:43.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.283473 7428 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:43.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.283694 7428 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34029/jwks.json
08:16:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:43.320432 7428 master_runner.cc:386] Master server non-default flags:
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.190.keytab
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.190
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.190:45779
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/info.pb
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.190
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/logs
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:43.321877 7428 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:43.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:43.323195 7428 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:43.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.342870 7435 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.343520 7434 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:43.345021 7428 server_base.cc:1034] running on GCE node
08:16:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:43.343693 7437 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:44.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.501317 7428 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.504065 7428 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.505523 7428 hybrid_clock.cc:648] HybridClock initialized: now 1714724204505474 us; error 49 us; skew 500 ppm
08:16:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.506417 7428 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:44.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:44 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724204, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:44.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.517799 7428 init.cc:377] Logged in from keytab as kudu/127.7.58.190@KRBTEST.COM (short username kudu)
08:16:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.523573 7428 webserver.cc:460] Webserver started at http://127.7.58.190:46145/ using document root <none> and password file <none>
08:16:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.524886 7428 fs_manager.cc:359] Metadata directory not provided
08:16:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.525138 7428 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:44.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.525626 7428 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.530984 7428 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/instance:
08:16:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72d56a27c33f4eed981bfee9726fff44"
08:16:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:44 on dist-test-slave-zl2t"
08:16:44.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.532166 7428 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal/instance:
08:16:44.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72d56a27c33f4eed981bfee9726fff44"
08:16:44.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:44 on dist-test-slave-zl2t"
08:16:44.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.539724 7428 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
08:16:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.545375 7446 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.546478 7428 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.546913 7428 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72d56a27c33f4eed981bfee9726fff44"
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:44 on dist-test-slave-zl2t"
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.547259 7428 fs_report.cc:383] FS layout report
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/data
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:44.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.609395 7428 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.617913 7428 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.618541 7428 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.704988 7428 rpc_server.cc:307] RPC server started. Bound to: 127.7.58.190:45779
08:16:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.705068 7497 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.58.190:45779 every 8 connection(s)
08:16:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.710094 7428 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/info.pb
08:16:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.718835 7402 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7428
08:16:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.719358 7402 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal/instance
08:16:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.719529 7498 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:44.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.732800 7498 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } has no permanent_uuid. Determining permanent_uuid...
08:16:44.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.58.189@KRBTEST.COM; defaulting to no policy
08:16:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.58.189@KRBTEST.COM" created.
08:16:44.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:44 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724204, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.190@KRBTEST.COM for kudu/127.7.58.190@KRBTEST.COM
08:16:44.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:44.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.58.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.189.keytab.
08:16:44.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.58.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.189.keytab.
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.774441 7402 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/logs
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/info.pb
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.189:37459
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.189
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:44.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.189.keytab
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.189
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:44.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:44.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.780732 7498 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } has no permanent_uuid. Determining permanent_uuid...
08:16:44.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:44.782980 7449 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:44.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:44.787947 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.189:37459: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.837441 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } attempt: 1
08:16:44.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:44.841323 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.189:37459: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:44.909744 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } attempt: 2
08:16:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:44.914269 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.189:37459: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:45.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:45.005721 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } attempt: 3
08:16:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.009637 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.189:37459: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.108937 7504 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.109493 7504 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:45.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.110066 7504 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:45.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.141227 7504 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:45.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.141489 7504 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.141877 7504 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.142102 7504 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.142431 7504 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:45.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.142730 7504 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:45.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.143105 7504 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34029/jwks.json
08:16:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:45.153105 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } attempt: 4
08:16:45.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.157054 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.189:37459: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:45.181946 7504 master_runner.cc:386] Master server non-default flags:
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.189.keytab
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.189
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.189:37459
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/info.pb
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.189
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/logs
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:45.183195 7504 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:45.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:45.184222 7504 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:45.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.202068 7511 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:45.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:45.456513 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } attempt: 5
08:16:45.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.461333 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.189:37459: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:46.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.008975 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } attempt: 6
08:16:46.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.016165 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.189:37459: Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.202448 7514 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:45.203142 7512 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:45.203806 7504 server_base.cc:1034] running on GCE node
08:16:46.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.367897 7504 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.371179 7504 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:46.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.372550 7504 hybrid_clock.cc:648] HybridClock initialized: now 1714724206372508 us; error 66 us; skew 500 ppm
08:16:46.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.373346 7504 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:46.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:46 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724206, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:46.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.379259 7504 init.cc:377] Logged in from keytab as kudu/127.7.58.189@KRBTEST.COM (short username kudu)
08:16:46.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.385141 7504 webserver.cc:460] Webserver started at http://127.7.58.189:46427/ using document root <none> and password file <none>
08:16:46.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.386020 7504 fs_manager.cc:359] Metadata directory not provided
08:16:46.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.386216 7504 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:46.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.386592 7504 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.391422 7504 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/instance:
08:16:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b26cd966bc1c4c648fbe11a4fb6c815e"
08:16:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:46 on dist-test-slave-zl2t"
08:16:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.392489 7504 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal/instance:
08:16:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b26cd966bc1c4c648fbe11a4fb6c815e"
08:16:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:46 on dist-test-slave-zl2t"
08:16:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.400314 7504 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
08:16:46.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.406484 7525 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:46.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.407596 7504 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:16:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.407967 7504 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b26cd966bc1c4c648fbe11a4fb6c815e"
08:16:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:46 on dist-test-slave-zl2t"
08:16:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.408310 7504 fs_report.cc:383] FS layout report
08:16:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:46.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/data
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:46.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.472472 7504 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.478138 7504 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:46.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.478588 7504 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:46.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.557070 7504 rpc_server.cc:307] RPC server started. Bound to: 127.7.58.189:37459
08:16:46.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.557163 7577 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.58.189:37459 every 8 connection(s)
08:16:46.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.559670 7504 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/info.pb
08:16:46.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.561964 7402 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7504
08:16:46.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.562415 7402 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal/instance
08:16:46.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.566320 7578 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:46.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.58.188@KRBTEST.COM; defaulting to no policy
08:16:46.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.580180 7578 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } has no permanent_uuid. Determining permanent_uuid...
08:16:46.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:46.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.58.188@KRBTEST.COM" created.
08:16:46.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:46 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724206, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.189@KRBTEST.COM for kudu/127.7.58.190@KRBTEST.COM
08:16:46.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.621016 7578 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } has no permanent_uuid. Determining permanent_uuid...
08:16:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:16:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.58.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.188.keytab.
08:16:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.58.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.188.keytab.
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.627183 7402 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:46.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/logs
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/info.pb
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.188:39345
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.188
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.188.keytab
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.188
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:46.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:46.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:46 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724206, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.189@KRBTEST.COM for kudu/127.7.58.189@KRBTEST.COM
08:16:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.653939 7578 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } has no permanent_uuid. Determining permanent_uuid...
08:16:46.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.656203 7526 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.660522 7578 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:46.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.712029 7578 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 1
08:16:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.717532 7578 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.786536 7578 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 2
08:16:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.790796 7578 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:46.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:46.882197 7578 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 3
08:16:46.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.885932 7578 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.970506 7585 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.971606 7585 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:46.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:46.972774 7585 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:47.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.015298 7585 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:47.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.015543 7585 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.015805 7585 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.016005 7585 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.016194 7585 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.016382 7585 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.016532 7585 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34029/jwks.json
08:16:47.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.029333 7578 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 4
08:16:47.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.033560 7578 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.054039 7585 master_runner.cc:386] Master server non-default flags:
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.188.keytab
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.188
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.188:39345
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/info.pb
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.188
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/logs
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.055279 7585 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:47.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.056296 7585 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:47.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.074395 7593 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.076809 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } attempt: 7
08:16:47.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:47 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724204, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.190@KRBTEST.COM for kudu/127.7.58.189@KRBTEST.COM
08:16:47.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.101562 7498 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } has no permanent_uuid. Determining permanent_uuid...
08:16:47.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.108832 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.167272 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 1
08:16:47.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.173466 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.254851 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 2
08:16:47.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.259822 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.332935 7578 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 5
08:16:47.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.337414 7578 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.345253 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 3
08:16:47.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.348961 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.489332 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 4
08:16:47.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.494345 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.777745 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 5
08:16:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.782491 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:47.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.884804 7578 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 6
08:16:47.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.891430 7578 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:48.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.074936 7592 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:48.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:47.076519 7585 server_base.cc:1034] running on GCE node
08:16:48.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:47.075330 7595 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:48.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.275661 7585 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:48.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.278935 7585 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:48.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.280386 7585 hybrid_clock.cc:648] HybridClock initialized: now 1714724208280337 us; error 61 us; skew 500 ppm
08:16:48.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.281436 7585 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:48.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724208, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:48.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.287976 7585 init.cc:377] Logged in from keytab as kudu/127.7.58.188@KRBTEST.COM (short username kudu)
08:16:48.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.293421 7585 webserver.cc:460] Webserver started at http://127.7.58.188:33675/ using document root <none> and password file <none>
08:16:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.294278 7585 fs_manager.cc:359] Metadata directory not provided
08:16:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.294468 7585 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.294891 7585 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:16:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.300012 7585 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/instance:
08:16:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53e94c1d94c5427f9fd918dd0944a310"
08:16:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:48 on dist-test-slave-zl2t"
08:16:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.301357 7585 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal/instance:
08:16:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53e94c1d94c5427f9fd918dd0944a310"
08:16:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:48 on dist-test-slave-zl2t"
08:16:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.309456 7585 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.001s
08:16:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.316223 7611 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.317307 7585 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
08:16:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.317600 7585 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53e94c1d94c5427f9fd918dd0944a310"
08:16:48.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:48 on dist-test-slave-zl2t"
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.317900 7585 fs_report.cc:383] FS layout report
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/data
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:48.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.334981 7498 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } attempt: 6
08:16:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:48.342397 7498 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.7.58.188:39345: Network error: Client connection negotiation failed: client connection to 127.7.58.188:39345: connect: Connection refused (error 111)
08:16:48.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.371884 7585 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.377718 7585 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:48.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.378247 7585 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:48.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.462291 7585 rpc_server.cc:307] RPC server started. Bound to: 127.7.58.188:39345
08:16:48.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.462378 7665 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.58.188:39345 every 8 connection(s)
08:16:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.465044 7585 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/info.pb
08:16:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.468714 7402 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7585
08:16:48.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.469193 7402 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal/instance
08:16:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.472370 7666 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:16:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.486030 7666 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } has no permanent_uuid. Determining permanent_uuid...
08:16:48.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724208, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.188@KRBTEST.COM for kudu/127.7.58.190@KRBTEST.COM
08:16:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724202, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.58.188@KRBTEST.COM
08:16:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.519416 7666 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } has no permanent_uuid. Determining permanent_uuid...
08:16:48.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724208, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.188@KRBTEST.COM for kudu/127.7.58.189@KRBTEST.COM
08:16:48.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724202, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.58.189@KRBTEST.COM
08:16:48.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.547578 7666 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } has no permanent_uuid. Determining permanent_uuid...
08:16:48.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724208, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.188@KRBTEST.COM for kudu/127.7.58.188@KRBTEST.COM
08:16:48.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724202, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.58.190@KRBTEST.COM
08:16:48.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.577942 7402 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
08:16:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.579372 7666 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Bootstrap starting.
08:16:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.584986 7666 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Neither blocks nor log segments found. Creating new log.
08:16:48.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:48.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:16:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.587167 7666 log.cc:824] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Log is configured to *not* fsync() on all Append() calls
08:16:48.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.592572 7666 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: No bootstrap required, opened a new log
08:16:48.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:16:48.605 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:16:48.609 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.58.190"
port: 45779
}
bound_http_address {
host: "127.7.58.190"
port: 46145
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.58.189"
port: 37459
}
bound_http_address {
host: "127.7.58.189"
port: 46427
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.58.188"
port: 39345
}
bound_http_address {
host: "127.7.58.188"
port: 33675
}
}
}
08:16:48.609 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:16:48.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:48.610 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:16:48.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:16:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.636560 7666 raft_consensus.cc:348] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.637382 7666 raft_consensus.cc:374] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:16:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.637667 7666 raft_consensus.cc:729] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53e94c1d94c5427f9fd918dd0944a310, State: Initialized, Role: FOLLOWER
08:16:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.638664 7666 consensus_queue.cc:260] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:48.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724208, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:48.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:16:48.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.642217 7677 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } } }
08:16:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.643330 7677 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [sys.catalog]: This master's current role is: FOLLOWER
08:16:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.643911 7666 sys_catalog.cc:564] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [sys.catalog]: configured and running, proceeding with master startup.
08:16:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.645788 7677 raft_consensus.cc:482] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.646260 7677 raft_consensus.cc:504] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:48.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.650329 7677 leader_election.cc:288] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779), b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459)
08:16:48.651 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:48.654 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc
08:16:48.660 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:16:48.661 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc
08:16:48.663 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:16:48.667 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:16:48.669 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:16:48.669 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:16:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.683940 7688 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:16:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:48 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724208, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:16:48.713 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:48.713 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc
08:16:48.715 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:16:48.716 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.58.189:37459
08:16:48.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:16:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.720232 7402 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 7504
08:16:48.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.689082 7585 master_runner.cc:186] Error getting master registration for 127.7.58.190:45779: OK, instance_id { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" instance_seqno: 1714724204681338 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:16:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.732057 7585 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:48.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:48.744354 7616 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.7.58.189:37459 (error 108)
08:16:48.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:48.750 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.58.190:45779
08:16:48.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:16:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:48.750564 7616 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459): Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:48.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.751420 7402 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 7428
08:16:48.772 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:48.773 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.58.188:39345
08:16:48.773 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:16:48.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.773874 7402 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 7585
08:16:48.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:48.796 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.58.189:37459
08:16:48.798 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:48.800258 7402 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/logs
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/info.pb
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.189:37459
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.189
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46427
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.189.keytab
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.189
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:48.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.115293 7692 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.115972 7692 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:49.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.116674 7692 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:49.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.149240 7692 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:49.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.149533 7692 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:49.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.149935 7692 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:49.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.150246 7692 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:49.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.150532 7692 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:49.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.150873 7692 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:49.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.151136 7692 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34029/jwks.json
08:16:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:49.188469 7692 master_runner.cc:386] Master server non-default flags:
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.189.keytab
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.189
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.189:37459
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/info.pb
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.189
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46427
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/logs
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:49.189613 7692 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:49.190565 7692 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:49.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.206280 7699 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.206353 7700 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:49.208138 7702 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:49.209316 7692 server_base.cc:1034] running on GCE node
08:16:50.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.364178 7692 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.367517 7692 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.369076 7692 hybrid_clock.cc:648] HybridClock initialized: now 1714724210369011 us; error 71 us; skew 500 ppm
08:16:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.369980 7692 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:50 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724210, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.382076 7692 init.cc:377] Logged in from keytab as kudu/127.7.58.189@KRBTEST.COM (short username kudu)
08:16:50.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.387485 7692 webserver.cc:460] Webserver started at http://127.7.58.189:46427/ using document root <none> and password file <none>
08:16:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.388442 7692 fs_manager.cc:359] Metadata directory not provided
08:16:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.388681 7692 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:50.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.396502 7692 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
08:16:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.401021 7711 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.402140 7692 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:16:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.402513 7692 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b26cd966bc1c4c648fbe11a4fb6c815e"
08:16:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:46 on dist-test-slave-zl2t"
08:16:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.404847 7692 fs_report.cc:383] FS layout report
08:16:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal
08:16:50.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/data
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:50.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.458444 7692 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.464469 7692 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.465003 7692 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.534404 7692 rpc_server.cc:307] RPC server started. Bound to: 127.7.58.189:37459
08:16:50.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.534520 7762 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.58.189:37459 every 8 connection(s)
08:16:50.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.537223 7692 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/data/info.pb
08:16:50.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.540796 7402 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7692
08:16:50.541 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:50.541 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.58.190:45779
08:16:50.541 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.543076 7402 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/logs
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/info.pb
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:50.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.190:45779
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.190
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46145
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.190.keytab
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.190
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:50.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:50.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:50.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:50.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:50.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.549906 7763 sys_catalog.cc:263] Verifying existing consensus state
08:16:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240503 08:16:50.550887 7763 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:50.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.551365 7692 master.cc:537] Master@127.7.58.189:37459 shutting down...
08:16:50.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.610931 7692 master.cc:552] Master@127.7.58.189:37459 shutdown complete.
08:16:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.875838 7765 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.876458 7765 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.877103 7765 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:50.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.908399 7765 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:50.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.908644 7765 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.909026 7765 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.909263 7765 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.909458 7765 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.909648 7765 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.909834 7765 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34029/jwks.json
08:16:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.948364 7765 master_runner.cc:386] Master server non-default flags:
08:16:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.190.keytab
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.190
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.190:45779
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/info.pb
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.190
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46145
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/logs
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.949565 7765 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.950635 7765 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.966070 7771 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:52.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.966405 7772 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:52.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:50.968156 7774 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:50.969035 7765 server_base.cc:1034] running on GCE node
08:16:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.120849 7765 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.123800 7765 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.125283 7765 hybrid_clock.cc:648] HybridClock initialized: now 1714724212125183 us; error 90 us; skew 500 ppm
08:16:52.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.126106 7765 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:52.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:52 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724212, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:52.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.132853 7765 init.cc:377] Logged in from keytab as kudu/127.7.58.190@KRBTEST.COM (short username kudu)
08:16:52.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.137550 7765 webserver.cc:460] Webserver started at http://127.7.58.190:46145/ using document root <none> and password file <none>
08:16:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.138468 7765 fs_manager.cc:359] Metadata directory not provided
08:16:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.138731 7765 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:52.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.147784 7765 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
08:16:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.152127 7785 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:52.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.153327 7765 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.153635 7765 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72d56a27c33f4eed981bfee9726fff44"
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:44 on dist-test-slave-zl2t"
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.156189 7765 fs_report.cc:383] FS layout report
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/data
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.203464 7765 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.209249 7765 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.209676 7765 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.284065 7765 rpc_server.cc:307] RPC server started. Bound to: 127.7.58.190:45779
08:16:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.284260 7837 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.58.190:45779 every 8 connection(s)
08:16:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.286659 7765 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/data/info.pb
08:16:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.289062 7402 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7765
08:16:52.290 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:52.291 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.58.188:39345
08:16:52.291 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.293237 7402 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/logs
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/info.pb
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.188:39345
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.188
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33675
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.188.keytab
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.188
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:16:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:16:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.298403 7838 sys_catalog.cc:263] Verifying existing consensus state
08:16:52.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240503 08:16:52.299247 7838 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:52.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.299688 7765 master.cc:537] Master@127.7.58.190:45779 shutting down...
08:16:52.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.362820 7765 master.cc:552] Master@127.7.58.190:45779 shutdown complete.
08:16:52.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:16:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.625653 7840 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:16:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.626411 7840 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:16:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.627156 7840 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:16:52.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.658041 7840 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:16:52.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.658330 7840 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:16:52.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.658775 7840 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:16:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.659081 7840 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:16:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.659377 7840 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:16:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.659665 7840 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:16:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.659896 7840 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34029/jwks.json
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.702139 7840 master_runner.cc:386] Master server non-default flags:
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.58.148:37025
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.58.190:45779,127.7.58.189:37459,127.7.58.188:39345
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/krb5kdc/kudu_127.7.58.188.keytab
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.58.188
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.58.188:39345
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34029/jwks.json
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/info.pb
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.58.188
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33675
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/logs
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:16:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c39e72a98e124e5a12080bac42105b996594d806
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 07:43:23 UTC on 1a15035a6830
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 104
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:16:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.703624 7840 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:16:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.704821 7840 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:16:52.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.724668 7847 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.725183 7846 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:52.726821 7840 server_base.cc:1034] running on GCE node
08:16:53.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:52.725586 7849 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:16:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.882090 7840 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:16:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.885331 7840 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:16:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.886783 7840 hybrid_clock.cc:648] HybridClock initialized: now 1714724213886739 us; error 58 us; skew 500 ppm
08:16:53.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.887553 7840 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:16:53.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:53 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724213, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:16:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.894380 7840 init.cc:377] Logged in from keytab as kudu/127.7.58.188@KRBTEST.COM (short username kudu)
08:16:53.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.899538 7840 webserver.cc:460] Webserver started at http://127.7.58.188:33675/ using document root <none> and password file <none>
08:16:53.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.900594 7840 fs_manager.cc:359] Metadata directory not provided
08:16:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.900864 7840 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:16:53.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.908366 7840 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
08:16:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.912686 7857 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:16:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.913655 7840 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
08:16:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.913980 7840 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data,/tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53e94c1d94c5427f9fd918dd0944a310"
08:16:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 08:16:48 on dist-test-slave-zl2t"
08:16:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.916292 7840 fs_report.cc:383] FS layout report
08:16:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:16:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/wal
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/data
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:16:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:16:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.970669 7840 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:16:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.978135 7840 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:16:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:53.978746 7840 kserver.cc:163] Server-wide thread pool size limit: 3276
08:16:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.052063 7840 rpc_server.cc:307] RPC server started. Bound to: 127.7.58.188:39345
08:16:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.052207 7909 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.58.188:39345 every 8 connection(s)
08:16:54.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.054890 7840 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskiFzj_p/test-tmp/mini-kudu-cluster8315630074409265337/master-2/data/info.pb
08:16:54.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.058729 7402 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu as pid 7840
08:16:54.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:16:54.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.068404 7910 sys_catalog.cc:263] Verifying existing consensus state
08:16:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.077050 7910 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Bootstrap starting.
08:16:54.113 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.123867 7910 log.cc:824] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Log is configured to *not* fsync() on all Append() calls
08:16:54.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.129871 7910 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:16:54.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.130909 7910 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Bootstrap complete.
08:16:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.167269 7910 raft_consensus.cc:348] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.170658 7910 raft_consensus.cc:374] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:16:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.170967 7910 raft_consensus.cc:729] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53e94c1d94c5427f9fd918dd0944a310, State: Initialized, Role: FOLLOWER
08:16:54.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.171938 7910 consensus_queue.cc:260] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:54.173 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.174 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.58.188:39345(127.7.58.188:39345)] connecting to peer
08:16:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.179533 7916 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } } }
08:16:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.180631 7916 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [sys.catalog]: This master's current role is: FOLLOWER
08:16:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.181275 7910 sys_catalog.cc:564] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [sys.catalog]: configured and running, proceeding with master startup.
08:16:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:54.225165 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:54.225503 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:54.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:54.238317 7863 tablet.cc:2369] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:16:54.264 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.268 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.271 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.58.188:39345(127.7.58.188:39345)] Successfully connected to peer
08:16:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.222977 7840 master_runner.cc:186] Error getting master registration for 127.7.58.190:45779: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111),
08:16:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.272361 7840 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:16:54.342 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:16:54.352 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.58.188
08:16:54.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.417995 7916 raft_consensus.cc:482] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:54.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.418560 7916 raft_consensus.cc:504] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:54.423866 7858 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111)
08:16:54.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.426934 7916 leader_election.cc:288] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779), b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459)
08:16:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:54.430755 7858 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779): Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111)
08:16:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:54.433321 7860 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459): Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.433763 7860 leader_election.cc:302] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53e94c1d94c5427f9fd918dd0944a310; no voters: 72d56a27c33f4eed981bfee9726fff44, b26cd966bc1c4c648fbe11a4fb6c815e
08:16:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:54.434991 7916 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:16:54 dist-test-slave-zl2t krb5kdc[7415](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724208, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.58.188@KRBTEST.COM
08:16:54.604 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x2ed472c2, L:/127.0.0.1:44440 - R:/127.7.58.188:39345] using SASL/GSSAPI
08:16:54.635 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:54.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.648 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.652 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.652 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.653 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.659 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:54.665 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.666 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.667 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.667 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.671 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.672 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.678 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:54.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.706 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.707 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.707 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.710 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.715 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.716 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.716 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.717 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:54.724 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.725 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.727 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.727 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.727 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:54.745 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.747 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.747 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.748 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.754 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.757 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:54.824 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.826 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.827 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.828 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.832 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.832 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.832 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.834 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:54.866 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:54.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:54.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:54.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.872 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:54.874 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:55.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:55.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:55.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.054 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.055 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:55.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:55.229833 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:55.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:55.230170 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:55.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:55.427 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:55.427 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.431 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.434 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:55.885 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:55.887 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:55.887 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.887 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:55.890 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:55.892 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:56.233489 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:56.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:56.233747 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:56.388777 7933 raft_consensus.cc:482] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:56.389143 7933 raft_consensus.cc:504] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:56.390487 7933 leader_election.cc:288] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779), b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459)
08:16:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:56.396677 7860 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459): Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:56.397071 7858 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779): Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111)
08:16:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:56.397485 7858 leader_election.cc:302] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53e94c1d94c5427f9fd918dd0944a310; no voters: 72d56a27c33f4eed981bfee9726fff44, b26cd966bc1c4c648fbe11a4fb6c815e
08:16:56.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:56.398109 7933 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:57.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:57.236510 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:57.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:57.236780 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:57.685 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:57.686 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:57.687 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:57.687 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:57.687 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:57.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:57.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:57.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:57.692 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:57.824 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:16:57.825 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:16:57.827 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:57.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:57.827 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:16:57.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:57.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:57.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:16:57.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:16:58.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:58.239817 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:58.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:58.240082 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:59.215039 7941 raft_consensus.cc:482] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:16:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:59.215402 7941 raft_consensus.cc:504] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:16:59.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:59.216758 7941 leader_election.cc:288] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779), b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459)
08:16:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:59.221854 7860 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459): Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:16:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:59.222333 7858 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779): Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111)
08:16:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:59.222723 7858 leader_election.cc:302] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53e94c1d94c5427f9fd918dd0944a310; no voters: 72d56a27c33f4eed981bfee9726fff44, b26cd966bc1c4c648fbe11a4fb6c815e
08:16:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:59.223294 7941 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:16:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:59.243479 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:16:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:16:59.243752 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:16:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:59.274394 7840 master_runner.cc:186] Error getting master registration for 127.7.58.190:45779: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111),
08:16:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:16:59.274806 7840 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:00.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:00.247102 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:00.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:00.247378 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:00.505 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:00.506 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:00.507 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:00.507 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:00.508 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:00.508 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:00.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:00.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:00.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:01.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:01.075416 7946 raft_consensus.cc:482] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:01.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:01.075815 7946 raft_consensus.cc:504] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:17:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:01.077371 7946 leader_election.cc:288] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779), b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459)
08:17:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:01.079695 7858 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111) [suppressed 5 similar messages]
08:17:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:01.082688 7860 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459): Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:17:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:01.083210 7858 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779): Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111)
08:17:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:01.083495 7858 leader_election.cc:302] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53e94c1d94c5427f9fd918dd0944a310; no voters: 72d56a27c33f4eed981bfee9726fff44, b26cd966bc1c4c648fbe11a4fb6c815e
08:17:01.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:01.084045 7946 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:01.250350 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:01.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:01.250689 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:02.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:02.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:02.047 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:02.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:02.048 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:02.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:02.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:02.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:02.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:02.253660 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:02.253921 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:03.145 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:03.146 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:03.147 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:03.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:03.148 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:03.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:03.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:03.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:03.153 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:03.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:03.257117 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:03.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:03.257433 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:03.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:17:03 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724223, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:03.898207 7853 init.cc:295] Successfully reacquired a new kerberos TGT
08:17:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:04.260133 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:04.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:04.260392 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:04.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:04.278415 7840 master_runner.cc:186] Error getting master registration for 127.7.58.190:45779: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111),
08:17:04.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:04.278873 7840 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:05.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:05.006 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:05.006 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:05.007 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:05.007 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:05.008 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:05.008 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:05.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:05.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:05.263128 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:05.263381 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:05.405 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:05.406 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:05.407 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:05.407 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:05.407 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:05.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:05.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:05.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:05.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:06.085 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:06.087 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:06.087 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:06.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:06.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:06.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:06.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:06.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:06.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:06.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:06.266337 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:06.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:06.266582 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:06.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:06.424813 7959 raft_consensus.cc:482] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:06.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:06.425325 7959 raft_consensus.cc:504] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:17:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:06.427045 7959 leader_election.cc:288] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779), b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459)
08:17:06.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:06.429304 7858 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111) [suppressed 1 similar messages]
08:17:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:06.432199 7860 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459): Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:17:06.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:06.432653 7858 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779): Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111)
08:17:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:06.433024 7858 leader_election.cc:302] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53e94c1d94c5427f9fd918dd0944a310; no voters: 72d56a27c33f4eed981bfee9726fff44, b26cd966bc1c4c648fbe11a4fb6c815e
08:17:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:06.433652 7959 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:07.269392 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:07.269670 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:07.605 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:07.606 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:07.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:07.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:07.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:07.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:07.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:07.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:07.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:08.272465 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:08.272727 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:09.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:09.275524 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:09.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:09.275825 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:09.282028 7840 master_runner.cc:186] Error getting master registration for 127.7.58.190:45779: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111),
08:17:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:09.282541 7840 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:10.278805 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:10.279070 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:10.585 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:10.585 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:10.586 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:10.587 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:10.587 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:10.587 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:10.587 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:10.588 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:10.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:11.281936 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:11.282272 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:11.525 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:11.526 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:11.527 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:11.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:11.527 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:11.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:11.528 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:11.528 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:11.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:11.585 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:11.586 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:11.586 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:11.586 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:11.586 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:11.587 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:11.587 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:11.588 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:11.589 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:12.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:12.285109 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:12.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:12.285398 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:12.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:12.614537 7971 raft_consensus.cc:482] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:12.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:12.615028 7971 raft_consensus.cc:504] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "72d56a27c33f4eed981bfee9726fff44" member_type: VOTER last_known_addr { host: "127.7.58.190" port: 45779 } } peers { permanent_uuid: "b26cd966bc1c4c648fbe11a4fb6c815e" member_type: VOTER last_known_addr { host: "127.7.58.189" port: 37459 } } peers { permanent_uuid: "53e94c1d94c5427f9fd918dd0944a310" member_type: VOTER last_known_addr { host: "127.7.58.188" port: 39345 } }
08:17:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:12.617054 7971 leader_election.cc:288] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779), b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459)
08:17:12.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:12.641595 7858 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111) [suppressed 1 similar messages]
08:17:12.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:12.645229 7858 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 72d56a27c33f4eed981bfee9726fff44 (127.7.58.190:45779): Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111)
08:17:12.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:12.645798 7860 leader_election.cc:334] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b26cd966bc1c4c648fbe11a4fb6c815e (127.7.58.189:37459): Network error: Client connection negotiation failed: client connection to 127.7.58.189:37459: connect: Connection refused (error 111)
08:17:12.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:12.646104 7860 leader_election.cc:302] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 53e94c1d94c5427f9fd918dd0944a310; no voters: 72d56a27c33f4eed981bfee9726fff44, b26cd966bc1c4c648fbe11a4fb6c815e
08:17:12.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:12.646982 7971 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:13.288764 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:13.289049 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 08:17:13 dist-test-slave-zl2t krb5kdc[7415](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714724233, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.58.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:13.900641 7853 init.cc:295] Successfully reacquired a new kerberos TGT
08:17:14.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:14.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:14.285343 7840 master_runner.cc:186] Error getting master registration for 127.7.58.190:45779: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111),
08:17:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:14.285799 7840 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:14.286 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:14.286 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:14.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:14.287 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:14.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:14.288 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:14.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:14.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:14.294771 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:14.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:14.295070 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:15.298307 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:15.298679 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:16.302001 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:16.302242 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:17.304960 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:17.305227 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:17.331974 6828 debug-util.cc:398] Leaking SignalData structure 0x7b08000afa60 after lost signal to thread 6763
08:17:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:17.332558 6828 debug-util.cc:398] Leaking SignalData structure 0x7b08000afce0 after lost signal to thread 6831
08:17:17.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:17.846 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:17.846 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:17.846 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:17.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:17.847 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:17.847 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:17.848 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:17.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:18.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:18.308109 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:18.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:18.308444 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:19.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:19.288529 7840 master_runner.cc:186] Error getting master registration for 127.7.58.190:45779: Network error: Client connection negotiation failed: client connection to 127.7.58.190:45779: connect: Connection refused (error 111),
08:17:19.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:19.288868 7840 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:19.311285 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:19.311549 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:19.525 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:19.526 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:19.527 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:19.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:19.527 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:19.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:19.528 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:19.528 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:19.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:20.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:20.314503 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:20.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:20.314864 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:20.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:20.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:20.486 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:20.486 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:20.486 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:20.487 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:20.487 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:20.487 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:20.489 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:21.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:21.317550 7928 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 53e94c1d94c5427f9fd918dd0944a310: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 08:17:21.317818 7928 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:22.225 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.189:37459(127.7.58.189:37459)] connecting to peer
08:17:22.226 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.58.190:45779(127.7.58.190:45779)] connecting to peer
08:17:22.226 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:22.226 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.58.189:37459(127.7.58.189:37459)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:22.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.189:37459: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459
08:17:22.227 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:22.227 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.190:45779(127.7.58.190:45779)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:22.227 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.58.190:45779: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
08:17:22.229 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345; will retry
08:17:22.234 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28166), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [198ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [196ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [551ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [567ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [588ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [589ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [591ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [598ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [602ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [609ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [638ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [646ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [648ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [658ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [664ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [678ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [758ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [763ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [765ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [796ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [800ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [803ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [805ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [979ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [984ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [986ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1356ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [1363ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1364ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [1365ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1816ms] refreshing cache from master, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [1818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [1818ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [1821ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3615ms] refreshing cache from master, [3616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [3616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [3618ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [3621ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3622ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [3623ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3755ms] refreshing cache from master, [3755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [3758ms] too many traces: truncated at 100 traces, deferred=Deferred@1566489333(state=PENDING, result=null, callback=(continuation of Deferred@945843128 after retry RPC after error@1571149737) -> (continuation of Deferred@811599999 after retry RPC after error@415339924) -> (continuation of Deferred@600244383 after retry RPC after error@1353483217) -> (continuation of Deferred@1623153227 after retry RPC after error@1959137021) -> (continuation of Deferred@1864252162 after retry RPC after error@1536724795) -> (continuation of Deferred@1097853697 after retry RPC after error@94020663) -> (continuation of Deferred@465721773 after retry RPC after error@79490233) -> (continuation of Deferred@976562380 after retry RPC after error@1792968865) -> (continuation of Deferred@1675148134 after retry RPC after error@6441600) -> (continuation of Deferred@327315720 after retry RPC after error@226981977) -> (continuation of Deferred@723710212 after retry RPC after error@956733610) -> (continuation of Deferred@965522611 after retry RPC after error@268387947) -> (continuation of Deferred@870923329 after retry RPC after error@1051140446) -> (continuation of Deferred@1657938709 after retry RPC after error@401341754) -> (continuation of Deferred@1830473906 after retry RPC after error@1789960606) -> (continuation of Deferred@428469590 after retry RPC after error@1363399024) -> (continuation of Deferred@1785136298 after retry RPC after error@87356457) -> (continuation of Deferred@1392593540 after retry RPC after error@478595525) -> (continuation of Deferred@1310117035 after retry RPC after error@1814162684) -> (continuation of Deferred@900583062 after retry RPC after error@533717255) -> (continuation of Deferred@1732515265 after retry RPC after error@81875929) -> (continuation of Deferred@1600382125 after retry RPC after error@1061478815) -> (continuation of Deferred@2028540331 after retry RPC after error@1981348596) -> (continuation of Deferred@916336150 after retry RPC after error@785283395) -> (continuation of Deferred@513429880 after retry RPC after error@795133701) -> (continuation of Deferred@1749922401 after retry RPC after error@890554954) -> (continuation of Deferred@2043239425 after retry RPC after error@1889329362), errback=(continuation of Deferred@945843128 after retry RPC after error@1571149737) -> (continuation of Deferred@811599999 after retry RPC after error@415339924) -> (continuation of Deferred@600244383 after retry RPC after error@1353483217) -> (continuation of Deferred@1623153227 after retry RPC after error@1959137021) -> (continuation of Deferred@1864252162 after retry RPC after error@1536724795) -> (continuation of Deferred@1097853697 after retry RPC after error@94020663) -> (continuation of Deferred@465721773 after retry RPC after error@79490233) -> (continuation of Deferred@976562380 after retry RPC after error@1792968865) -> (continuation of Deferred@1675148134 after retry RPC after error@6441600) -> (continuation of Deferred@327315720 after retry RPC after error@226981977) -> (continuation of Deferred@723710212 after retry RPC after error@956733610) -> (continuation of Deferred@965522611 after retry RPC after error@268387947) -> (continuation of Deferred@870923329 after retry RPC after error@1051140446) -> (continuation of Deferred@1657938709 after retry RPC after error@401341754) -> (continuation of Deferred@1830473906 after retry RPC after error@1789960606) -> (continuation of Deferred@428469590 after retry RPC after error@1363399024) -> (continuation of Deferred@1785136298 after retry RPC after error@87356457) -> (continuation of Deferred@1392593540 after retry RPC after error@478595525) -> (continuation of Deferred@1310117035 after retry RPC after error@1814162684) -> (continuation of Deferred@900583062 after retry RPC after error@533717255) -> (continuation of Deferred@1732515265 after retry RPC after error@81875929) -> (continuation of Deferred@1600382125 after retry RPC after error@1061478815) -> (continuation of Deferred@2028540331 after retry RPC after error@1981348596) -> (continuation of Deferred@916336150 after retry RPC after error@785283395) -> (continuation of Deferred@513429880 after retry RPC after error@795133701) -> (continuation of Deferred@1749922401 after retry RPC after error@890554954) -> (continuation of Deferred@2043239425 after retry RPC after error@1889329362)))
08:17:22.239 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.58.190:45779(127.7.58.190:45779)] disconnecting while in state TERMINATED
08:17:22.240 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.58.189:37459(127.7.58.189:37459)] disconnecting while in state TERMINATED
08:17:22.240 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.58.188:39345(127.7.58.188:39345)] disconnecting while in state READY
08:17:22.247 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:17:22.250 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.58.188:39345(127.7.58.188:39345)] handling channelInactive
08:17:22.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.58.188:39345(127.7.58.188:39345)] cleaning up while in state READY due to: connection closed
08:17:22.252 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:17:22.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:22.261281 7402 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 7840
08:17:23.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T08:17:23Z chronyd exiting
08:17:24.385 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:17:24.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:24.386734 6677 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 6956
08:17:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:24.411604 6677 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 7116
08:17:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:24.436861 6677 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 7258
08:17:24.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:24.465767 6677 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 6689
08:17:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:24.495756 6677 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 6759
08:17:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 08:17:24.528017 6677 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskiFzj_p/build/tsan/bin/kudu with pid 6834
08:17:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T08:17:25Z chronyd exiting
08:17:26.649 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28166), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [198ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [196ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [551ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [567ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [588ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [589ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [591ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [598ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [602ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [609ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [638ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [646ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [648ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [658ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [664ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [678ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [758ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [763ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [765ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [796ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [800ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [803ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [805ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [979ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [984ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [986ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1356ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [1363ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1364ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [1365ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1816ms] refreshing cache from master, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [1818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [1818ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [1821ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3615ms] refreshing cache from master, [3616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [3616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [3618ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [3621ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3622ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [3623ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3755ms] refreshing cache from master, [3755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [3758ms] too many traces: truncated at 100 traces, deferred=Deferred@1566489333(state=PENDING, result=null, callback=(continuation of Deferred@945843128 after retry RPC after error@1571149737) -> (continuation of Deferred@811599999 after retry RPC after error@415339924) -> (continuation of Deferred@600244383 after retry RPC after error@1353483217) -> (continuation of Deferred@1623153227 after retry RPC after error@1959137021) -> (continuation of Deferred@1864252162 after retry RPC after error@1536724795) -> (continuation of Deferred@1097853697 after retry RPC after error@94020663) -> (continuation of Deferred@465721773 after retry RPC after error@79490233) -> (continuation of Deferred@976562380 after retry RPC after error@1792968865) -> (continuation of Deferred@1675148134 after retry RPC after error@6441600) -> (continuation of Deferred@327315720 after retry RPC after error@226981977) -> (continuation of Deferred@723710212 after retry RPC after error@956733610) -> (continuation of Deferred@965522611 after retry RPC after error@268387947) -> (continuation of Deferred@870923329 after retry RPC after error@1051140446) -> (continuation of Deferred@1657938709 after retry RPC after error@401341754) -> (continuation of Deferred@1830473906 after retry RPC after error@1789960606) -> (continuation of Deferred@428469590 after retry RPC after error@1363399024) -> (continuation of Deferred@1785136298 after retry RPC after error@87356457) -> (continuation of Deferred@1392593540 after retry RPC after error@478595525) -> (continuation of Deferred@1310117035 after retry RPC after error@1814162684) -> (continuation of Deferred@900583062 after retry RPC after error@533717255) -> (continuation of Deferred@1732515265 after retry RPC after error@81875929) -> (continuation of Deferred@1600382125 after retry RPC after error@1061478815) -> (continuation of Deferred@2028540331 after retry RPC after error@1981348596) -> (continuation of Deferred@916336150 after retry RPC after error@785283395) -> (continuation of Deferred@513429880 after retry RPC after error@795133701) -> (continuation of Deferred@1749922401 after retry RPC after error@890554954) -> (continuation of Deferred@2043239425 after retry RPC after error@1889329362), errback=(continuation of Deferred@945843128 after retry RPC after error@1571149737) -> (continuation of Deferred@811599999 after retry RPC after error@415339924) -> (continuation of Deferred@600244383 after retry RPC after error@1353483217) -> (continuation of Deferred@1623153227 after retry RPC after error@1959137021) -> (continuation of Deferred@1864252162 after retry RPC after error@1536724795) -> (continuation of Deferred@1097853697 after retry RPC after error@94020663) -> (continuation of Deferred@465721773 after retry RPC after error@79490233) -> (continuation of Deferred@976562380 after retry RPC after error@1792968865) -> (continuation of Deferred@1675148134 after retry RPC after error@6441600) -> (continuation of Deferred@327315720 after retry RPC after error@226981977) -> (continuation of Deferred@723710212 after retry RPC after error@956733610) -> (continuation of Deferred@965522611 after retry RPC after error@268387947) -> (continuation of Deferred@870923329 after retry RPC after error@1051140446) -> (continuation of Deferred@1657938709 after retry RPC after error@401341754) -> (continuation of Deferred@1830473906 after retry RPC after error@1789960606) -> (continuation of Deferred@428469590 after retry RPC after error@1363399024) -> (continuation of Deferred@1785136298 after retry RPC after error@87356457) -> (continuation of Deferred@1392593540 after retry RPC after error@478595525) -> (continuation of Deferred@1310117035 after retry RPC after error@1814162684) -> (continuation of Deferred@900583062 after retry RPC after error@533717255) -> (continuation of Deferred@1732515265 after retry RPC after error@81875929) -> (continuation of Deferred@1600382125 after retry RPC after error@1061478815) -> (continuation of Deferred@2028540331 after retry RPC after error@1981348596) -> (continuation of Deferred@916336150 after retry RPC after error@785283395) -> (continuation of Deferred@513429880 after retry RPC after error@795133701) -> (continuation of Deferred@1749922401 after retry RPC after error@890554954) -> (continuation of Deferred@2043239425 after retry RPC after error@1889329362)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.58.190:45779
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28166), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [198ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [196ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [551ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [567ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [588ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [589ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [591ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [598ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [602ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [609ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [638ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [646ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [647ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [648ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [658ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [660ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [664ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [678ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [686ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [758ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [763ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [765ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [796ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [800ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [803ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [805ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [979ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [984ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [985ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [986ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1356ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [1363ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1364ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [1365ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1816ms] refreshing cache from master, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [1817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [1818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [1818ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [1821ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3615ms] refreshing cache from master, [3616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [3616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [3618ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.189:37459: Network error: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459, [3621ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.190:45779: Network error: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3622ms] Sub RPC ConnectToMaster: received response from server master-127.7.58.188:39345: OK, [3623ms] delaying RPC due to: Service unavailable: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779, [3755ms] refreshing cache from master, [3755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.189:37459, [3756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.190:45779, [3757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.58.188:39345, [3758ms] too many traces: truncated at 100 traces, deferred=Deferred@1566489333(state=PENDING, result=null, callback=(continuation of Deferred@945843128 after retry RPC after error@1571149737) -> (continuation of Deferred@811599999 after retry RPC after error@415339924) -> (continuation of Deferred@600244383 after retry RPC after error@1353483217) -> (continuation of Deferred@1623153227 after retry RPC after error@1959137021) -> (continuation of Deferred@1864252162 after retry RPC after error@1536724795) -> (continuation of Deferred@1097853697 after retry RPC after error@94020663) -> (continuation of Deferred@465721773 after retry RPC after error@79490233) -> (continuation of Deferred@976562380 after retry RPC after error@1792968865) -> (continuation of Deferred@1675148134 after retry RPC after error@6441600) -> (continuation of Deferred@327315720 after retry RPC after error@226981977) -> (continuation of Deferred@723710212 after retry RPC after error@956733610) -> (continuation of Deferred@965522611 after retry RPC after error@268387947) -> (continuation of Deferred@870923329 after retry RPC after error@1051140446) -> (continuation of Deferred@1657938709 after retry RPC after error@401341754) -> (continuation of Deferred@1830473906 after retry RPC after error@1789960606) -> (continuation of Deferred@428469590 after retry RPC after error@1363399024) -> (continuation of Deferred@1785136298 after retry RPC after error@87356457) -> (continuation of Deferred@1392593540 after retry RPC after error@478595525) -> (continuation of Deferred@1310117035 after retry RPC after error@1814162684) -> (continuation of Deferred@900583062 after retry RPC after error@533717255) -> (continuation of Deferred@1732515265 after retry RPC after error@81875929) -> (continuation of Deferred@1600382125 after retry RPC after error@1061478815) -> (continuation of Deferred@2028540331 after retry RPC after error@1981348596) -> (continuation of Deferred@916336150 after retry RPC after error@785283395) -> (continuation of Deferred@513429880 after retry RPC after error@795133701) -> (continuation of Deferred@1749922401 after retry RPC after error@890554954) -> (continuation of Deferred@2043239425 after retry RPC after error@1889329362), errback=(continuation of Deferred@945843128 after retry RPC after error@1571149737) -> (continuation of Deferred@811599999 after retry RPC after error@415339924) -> (continuation of Deferred@600244383 after retry RPC after error@1353483217) -> (continuation of Deferred@1623153227 after retry RPC after error@1959137021) -> (continuation of Deferred@1864252162 after retry RPC after error@1536724795) -> (continuation of Deferred@1097853697 after retry RPC after error@94020663) -> (continuation of Deferred@465721773 after retry RPC after error@79490233) -> (continuation of Deferred@976562380 after retry RPC after error@1792968865) -> (continuation of Deferred@1675148134 after retry RPC after error@6441600) -> (continuation of Deferred@327315720 after retry RPC after error@226981977) -> (continuation of Deferred@723710212 after retry RPC after error@956733610) -> (continuation of Deferred@965522611 after retry RPC after error@268387947) -> (continuation of Deferred@870923329 after retry RPC after error@1051140446) -> (continuation of Deferred@1657938709 after retry RPC after error@401341754) -> (continuation of Deferred@1830473906 after retry RPC after error@1789960606) -> (continuation of Deferred@428469590 after retry RPC after error@1363399024) -> (continuation of Deferred@1785136298 after retry RPC after error@87356457) -> (continuation of Deferred@1392593540 after retry RPC after error@478595525) -> (continuation of Deferred@1310117035 after retry RPC after error@1814162684) -> (continuation of Deferred@900583062 after retry RPC after error@533717255) -> (continuation of Deferred@1732515265 after retry RPC after error@81875929) -> (continuation of Deferred@1600382125 after retry RPC after error@1061478815) -> (continuation of Deferred@2028540331 after retry RPC after error@1981348596) -> (continuation of Deferred@916336150 after retry RPC after error@785283395) -> (continuation of Deferred@513429880 after retry RPC after error@795133701) -> (continuation of Deferred@1749922401 after retry RPC after error@890554954) -> (continuation of Deferred@2043239425 after retry RPC after error@1889329362)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.58.189:37459,127.7.58.190:45779,127.7.58.188:39345) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.189:37459(127.7.58.189:37459): Connection refused: /127.7.58.189:37459,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.58.190:45779(127.7.58.190:45779): Connection refused: /127.7.58.190:45779
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.58.190:45779
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)