13:39:20.340 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
13:39:20.351 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskPWbCo1/test-tmp
13:39:20.353 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:39:20.357 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskPWbCo1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:39:20.555 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
13:39:20.790 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:39:20.791 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:39:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:20.796589 28352 mini_oidc.cc:152] Starting JWKS server
13:39:20.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:20.800329 28352 webserver.cc:460] Webserver started at http://127.0.0.1:39287/ using document root <none> and password file <none>
13:39:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:20.800771 28352 mini_oidc.cc:189] Starting OIDC Discovery server
13:39:20.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:20.802872 28352 webserver.cc:460] Webserver started at http://127.0.0.1:40003/ using document root <none> and password file <none>
13:39:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:39:20Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:39:20.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:39:20Z Disabled control of system clock
13:39:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:20.864115 28352 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/wal
13:39:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/logs
13:39:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data/info.pb
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.62:40085
13:39:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.62
13:39:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39287/jwks.json with env {}
13:39:21.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.146206 28364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.146736 28364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.147414 28364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.176599 28364 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.176888 28364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.177078 28364 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.177269 28364 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.177525 28364 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:21.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.177745 28364 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39287/jwks.json
13:39:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:21.211262 28364 master_runner.cc:386] Master server non-default flags:
13:39:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data
13:39:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/wal
13:39:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.62:40085
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39287/jwks.json
13:39:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data/info.pb
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.62
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/logs
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:21.212499 28364 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:21.214506 28364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.230473 28370 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:22.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.230561 28371 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:22.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:21.232178 28373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:22.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:21.232862 28364 server_base.cc:1034] running on GCE node
13:39:22.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.263643 28364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:22.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.266682 28364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:22.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.268096 28364 hybrid_clock.cc:648] HybridClock initialized: now 1714138762268056 us; error 56 us; skew 500 ppm
13:39:22.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.268930 28364 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:22.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.275194 28364 webserver.cc:460] Webserver started at http://127.27.176.62:40051/ using document root <none> and password file <none>
13:39:22.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.276041 28364 fs_manager.cc:359] Metadata directory not provided
13:39:22.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.276234 28364 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:22.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.276644 28364 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.281291 28364 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data/instance:
13:39:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a03772693929407ab969350af3b6f461"
13:39:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:22 on dist-test-slave-d293"
13:39:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.282392 28364 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/wal/instance:
13:39:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a03772693929407ab969350af3b6f461"
13:39:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:22 on dist-test-slave-d293"
13:39:22.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.289182 28364 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
13:39:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.294535 28380 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.295495 28364 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
13:39:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.295832 28364 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/wal
13:39:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a03772693929407ab969350af3b6f461"
13:39:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:22 on dist-test-slave-d293"
13:39:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.296188 28364 fs_report.cc:383] FS layout report
13:39:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/wal
13:39:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/wal
13:39:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data/data
13:39:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:22.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.348224 28364 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.349887 28364 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.350419 28364 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.417129 28364 rpc_server.cc:307] RPC server started. Bound to: 127.27.176.62:40085
13:39:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.417233 28431 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.176.62:40085 every 8 connection(s)
13:39:22.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.419842 28364 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/data/info.pb
13:39:22.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.425078 28432 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:39:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.426026 28352 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 28364
13:39:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.426451 28352 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-0/wal/instance
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.429867 28352 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/wal
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/logs
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data/info.pb
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.61:41509
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.61
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:22.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:22.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39287/jwks.json with env {}
13:39:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.438899 28432 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } has no permanent_uuid. Determining permanent_uuid...
13:39:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.464571 28432 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } has no permanent_uuid. Determining permanent_uuid...
13:39:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.466867 28382 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:22.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.470803 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.61:41509: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.520376 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } attempt: 1
13:39:22.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.524788 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.61:41509: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:22.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.593214 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } attempt: 2
13:39:22.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.597051 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.61:41509: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:22.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.688412 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } attempt: 3
13:39:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.693153 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.61:41509: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.726586 28434 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.727131 28434 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.727675 28434 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.756714 28434 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.756994 28434 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.757185 28434 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.757375 28434 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.757584 28434 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.757719 28434 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39287/jwks.json
13:39:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.789925 28434 master_runner.cc:386] Master server non-default flags:
13:39:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data
13:39:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/wal
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.61:41509
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39287/jwks.json
13:39:22.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data/info.pb
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.61
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/logs
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:22.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:22.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:22.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:22.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:22.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:22.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:22.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:22.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.791065 28434 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:22.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.792690 28434 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:22.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.806766 28443 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:22.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.836586 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } attempt: 4
13:39:22.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.840212 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.61:41509: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.139724 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } attempt: 5
13:39:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:23.143697 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.61:41509: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.691183 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } attempt: 6
13:39:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:23.702739 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.61:41509: Network error: Client connection negotiation failed: client connection to 127.27.176.61:41509: connect: Connection refused (error 111)
13:39:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.806883 28444 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:22.807138 28446 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:22.809031 28434 server_base.cc:1034] running on GCE node
13:39:23.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.833812 28434 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:23.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.836843 28434 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.838291 28434 hybrid_clock.cc:648] HybridClock initialized: now 1714138763838249 us; error 57 us; skew 500 ppm
13:39:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.839273 28434 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.845296 28434 webserver.cc:460] Webserver started at http://127.27.176.61:42895/ using document root <none> and password file <none>
13:39:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.846182 28434 fs_manager.cc:359] Metadata directory not provided
13:39:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.846361 28434 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:23.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.846724 28434 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.851804 28434 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data/instance:
13:39:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "00dd22677867449f844dca31954804bf"
13:39:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:23 on dist-test-slave-d293"
13:39:23.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.853035 28434 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/wal/instance:
13:39:23.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "00dd22677867449f844dca31954804bf"
13:39:23.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:23 on dist-test-slave-d293"
13:39:23.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.860344 28434 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
13:39:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.865904 28455 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.866909 28434 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
13:39:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.867305 28434 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/wal
13:39:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "00dd22677867449f844dca31954804bf"
13:39:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:23 on dist-test-slave-d293"
13:39:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.867667 28434 fs_report.cc:383] FS layout report
13:39:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/wal
13:39:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/wal
13:39:23.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data/data
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:23.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.924671 28434 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.926069 28434 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.926476 28434 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:23.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.992431 28434 rpc_server.cc:307] RPC server started. Bound to: 127.27.176.61:41509
13:39:23.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.992563 28506 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.176.61:41509 every 8 connection(s)
13:39:23.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.994951 28434 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/data/info.pb
13:39:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.996167 28352 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 28434
13:39:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.996672 28352 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-1/wal/instance
13:39:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:23.999176 28352 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/wal
13:39:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data
13:39:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:23.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/logs
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data/info.pb
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:24.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.60:45615
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.60
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39287/jwks.json with env {}
13:39:24.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.001111 28507 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:39:24.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.012979 28507 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } has no permanent_uuid. Determining permanent_uuid...
13:39:24.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.033126 28507 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } has no permanent_uuid. Determining permanent_uuid...
13:39:24.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.046759 28507 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } has no permanent_uuid. Determining permanent_uuid...
13:39:24.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.049054 28458 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.053205 28507 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.102622 28507 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 1
13:39:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.106287 28507 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.174728 28507 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 2
13:39:24.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.179246 28507 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.270695 28507 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 3
13:39:24.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.274417 28507 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.290153 28509 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.290747 28509 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:24.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.291473 28509 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:24.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.320519 28509 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:24.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.320787 28509 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:24.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.320974 28509 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:24.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.321153 28509 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:24.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.321437 28509 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:24.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.321692 28509 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39287/jwks.json
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.357811 28509 master_runner.cc:386] Master server non-default flags:
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/wal
13:39:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.60:45615
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39287/jwks.json
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data/info.pb
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.60
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/logs
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:24.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.361898 28509 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.364454 28509 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:24.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.379689 28520 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:24.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.417825 28507 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 4
13:39:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.421463 28507 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.720871 28507 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 5
13:39:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.725773 28507 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.763223 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } attempt: 7
13:39:24.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.787886 28432 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } has no permanent_uuid. Determining permanent_uuid...
13:39:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.793330 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.851693 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 1
13:39:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.855849 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:24.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.937213 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 2
13:39:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.941171 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:25.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.026508 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 3
13:39:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:25.030575 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:25.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.170998 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 4
13:39:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:25.175076 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.273226 28507 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 6
13:39:25.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:25.279662 28507 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.379805 28519 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:25.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:24.379701 28522 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:24.381881 28509 server_base.cc:1034] running on GCE node
13:39:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.425879 28509 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.428550 28509 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.429915 28509 hybrid_clock.cc:648] HybridClock initialized: now 1714138765429883 us; error 40 us; skew 500 ppm
13:39:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.430749 28509 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.436669 28509 webserver.cc:460] Webserver started at http://127.27.176.60:43619/ using document root <none> and password file <none>
13:39:25.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.437589 28509 fs_manager.cc:359] Metadata directory not provided
13:39:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.437799 28509 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.438304 28509 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.443337 28509 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data/instance:
13:39:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdfb3438cc47426fa7787f24ccd7af7d"
13:39:25.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:25 on dist-test-slave-d293"
13:39:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.444530 28509 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/wal/instance:
13:39:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdfb3438cc47426fa7787f24ccd7af7d"
13:39:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:25 on dist-test-slave-d293"
13:39:25.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.451956 28509 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
13:39:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.457648 28534 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:25.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.458576 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 5
13:39:25.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.458822 28509 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
13:39:25.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.459198 28509 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/wal
13:39:25.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdfb3438cc47426fa7787f24ccd7af7d"
13:39:25.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:25 on dist-test-slave-d293"
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.459632 28509 fs_report.cc:383] FS layout report
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/wal
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/wal
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data/data
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:25.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:25.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:25.462505 28432 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.27.176.60:45615: Network error: Client connection negotiation failed: client connection to 127.27.176.60:45615: connect: Connection refused (error 111)
13:39:25.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.512167 28509 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:25.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.513638 28509 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:25.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.514041 28509 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:25.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.581297 28509 rpc_server.cc:307] RPC server started. Bound to: 127.27.176.60:45615
13:39:25.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.581398 28585 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.176.60:45615 every 8 connection(s)
13:39:25.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.583853 28509 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/data/info.pb
13:39:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.588784 28586 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:39:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.593299 28352 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 28509
13:39:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.593698 28352 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/master-2/wal/instance
13:39:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.600625 28586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } has no permanent_uuid. Determining permanent_uuid...
13:39:25.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.633978 28586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } has no permanent_uuid. Determining permanent_uuid...
13:39:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.642737 28586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } has no permanent_uuid. Determining permanent_uuid...
13:39:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.656849 28586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d: Bootstrap starting.
13:39:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.662528 28586 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d: Neither blocks nor log segments found. Creating new log.
13:39:25.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.664165 28586 log.cc:824] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d: Log is configured to *not* fsync() on all Append() calls
13:39:25.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.668826 28586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d: No bootstrap required, opened a new log
13:39:25.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.688356 28586 raft_consensus.cc:348] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.688982 28586 raft_consensus.cc:374] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:39:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.689163 28586 raft_consensus.cc:729] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdfb3438cc47426fa7787f24ccd7af7d, State: Initialized, Role: FOLLOWER
13:39:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.689807 28586 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.692493 28595 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.693111 28595 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: This master's current role is: FOLLOWER
13:39:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.694356 28586 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: configured and running, proceeding with master startup.
13:39:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:25.721009 28606 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:25.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:25.721380 28606 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:25.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.715679 28509 master_runner.cc:186] Error getting master registration for 127.27.176.62:40085: OK, instance_id { permanent_uuid: "a03772693929407ab969350af3b6f461" instance_seqno: 1714138762398397 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:39:25.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.738658 28509 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:39:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.757056 28595 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:39:25.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.757465 28595 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:25.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:25.759349 28595 leader_election.cc:288] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a03772693929407ab969350af3b6f461 (127.27.176.62:40085), 00dd22677867449f844dca31954804bf (127.27.176.61:41509)
13:39:25.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:25.768852 28540 tablet.cc:2369] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:39:26.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.014847 28432 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 6
13:39:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.035137 28432 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: Bootstrap starting.
13:39:26.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.041805 28432 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: Neither blocks nor log segments found. Creating new log.
13:39:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.044466 28432 log.cc:824] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: Log is configured to *not* fsync() on all Append() calls
13:39:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.049418 28432 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: No bootstrap required, opened a new log
13:39:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.080556 28432 raft_consensus.cc:348] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.081763 28432 raft_consensus.cc:374] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:39:26.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.082304 28432 raft_consensus.cc:729] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a03772693929407ab969350af3b6f461, State: Initialized, Role: FOLLOWER
13:39:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.083899 28432 consensus_queue.cc:260] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:26.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.086916 28610 sys_catalog.cc:455] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.087749 28610 sys_catalog.cc:458] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [sys.catalog]: This master's current role is: FOLLOWER
13:39:26.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.088760 28432 sys_catalog.cc:564] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [sys.catalog]: configured and running, proceeding with master startup.
13:39:26.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.088938 28407 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a03772693929407ab969350af3b6f461" is_pre_election: true
13:39:26.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.090050 28407 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bdfb3438cc47426fa7787f24ccd7af7d in term 0.
13:39:26.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.091332 28537 leader_election.cc:302] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a03772693929407ab969350af3b6f461, bdfb3438cc47426fa7787f24ccd7af7d; no voters:
13:39:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.092518 28595 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Leader pre-election won for term 1
13:39:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.092865 28595 raft_consensus.cc:482] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:39:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.093192 28595 raft_consensus.cc:3037] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 0 FOLLOWER]: Advancing to term 1
13:39:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.099557 28595 raft_consensus.cc:504] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:26.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.104184 28595 leader_election.cc:288] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [CANDIDATE]: Term 1 election: Requested vote from peers a03772693929407ab969350af3b6f461 (127.27.176.62:40085), 00dd22677867449f844dca31954804bf (127.27.176.61:41509)
13:39:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.104576 28407 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a03772693929407ab969350af3b6f461"
13:39:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.105122 28407 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [term 0 FOLLOWER]: Advancing to term 1
13:39:26.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.108769 28386 tablet.cc:2369] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:39:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.116880 28407 raft_consensus.cc:2454] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdfb3438cc47426fa7787f24ccd7af7d in term 1.
13:39:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.118052 28537 leader_election.cc:302] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a03772693929407ab969350af3b6f461, bdfb3438cc47426fa7787f24ccd7af7d; no voters:
13:39:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.118885 28595 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 1 FOLLOWER]: Leader election won for term 1
13:39:26.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.120898 28595 raft_consensus.cc:686] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [term 1 LEADER]: Becoming Leader. State: Replica: bdfb3438cc47426fa7787f24ccd7af7d, State: Running, Role: LEADER
13:39:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.122583 28595 consensus_queue.cc:237] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.136521 28618 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdfb3438cc47426fa7787f24ccd7af7d. Latest consensus state: current_term: 1 leader_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.137379 28618 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: This master's current role is: LEADER
13:39:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.140296 28621 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:39:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.149112 28621 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:39:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.191484 28624 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:26.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.194430 28624 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:26.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.209035 28407 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [term 1 FOLLOWER]: Refusing update from remote peer bdfb3438cc47426fa7787f24ccd7af7d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:39:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.211172 28618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [LEADER]: Connected to new peer: Peer: permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:39:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.163184 28364 master_runner.cc:186] Error getting master registration for 127.27.176.61:41509: OK, instance_id { permanent_uuid: "00dd22677867449f844dca31954804bf" instance_seqno: 1714138763973566 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:39:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.221947 28364 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:39:26.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.259410 28610 sys_catalog.cc:455] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdfb3438cc47426fa7787f24ccd7af7d. Latest consensus state: current_term: 1 leader_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.261381 28610 sys_catalog.cc:458] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [sys.catalog]: This master's current role is: FOLLOWER
13:39:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.265686 28595 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.266434 28595 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: This master's current role is: LEADER
13:39:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.268932 28621 catalog_manager.cc:1347] Generated new cluster ID: 39db064ae6df4c52ae243a0d3f56fc85
13:39:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.269217 28621 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:39:26.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.286024 28610 sys_catalog.cc:455] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.286759 28610 sys_catalog.cc:458] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461 [sys.catalog]: This master's current role is: FOLLOWER
13:39:26.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.299319 28621 catalog_manager.cc:1370] Generated new certificate authority record
13:39:26.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.305109 28621 catalog_manager.cc:1504] Loading token signing keys...
13:39:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.337801 28621 catalog_manager.cc:5857] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d: Generated new TSK 0
13:39:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.338910 28621 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:39:26.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.340029 28507 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } attempt: 7
13:39:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.368383 28352 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/wal
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/logs
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data/info.pb
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.1:0
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.176.1
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.1
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:39:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.391422 28507 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf: Bootstrap starting.
13:39:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.399168 28507 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf: Neither blocks nor log segments found. Creating new log.
13:39:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.402045 28507 log.cc:824] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf: Log is configured to *not* fsync() on all Append() calls
13:39:26.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.413120 28507 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf: No bootstrap required, opened a new log
13:39:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.443131 28507 raft_consensus.cc:348] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.444307 28507 raft_consensus.cc:374] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:39:26.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.444818 28507 raft_consensus.cc:729] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 00dd22677867449f844dca31954804bf, State: Initialized, Role: FOLLOWER
13:39:26.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.445966 28507 consensus_queue.cc:260] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } }
13:39:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.454222 28635 sys_catalog.cc:455] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.457763 28635 sys_catalog.cc:458] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [sys.catalog]: This master's current role is: FOLLOWER
13:39:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.458271 28481 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "00dd22677867449f844dca31954804bf"
13:39:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.458871 28481 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [term 0 FOLLOWER]: Advancing to term 1
13:39:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.457602 28482 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "00dd22677867449f844dca31954804bf" is_pre_election: true
13:39:26.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.457440 28507 sys_catalog.cc:564] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [sys.catalog]: configured and running, proceeding with master startup.
13:39:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.466279 28481 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdfb3438cc47426fa7787f24ccd7af7d in term 1.
13:39:26.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.467660 28480 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [term 1 FOLLOWER]: Refusing update from remote peer bdfb3438cc47426fa7787f24ccd7af7d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:39:26.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.475438 28595 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [LEADER]: Connected to new peer: Peer: permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:39:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.561762 28642 sys_catalog.cc:455] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.564195 28642 sys_catalog.cc:458] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [sys.catalog]: This master's current role is: FOLLOWER
13:39:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.563439 28635 sys_catalog.cc:455] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader bdfb3438cc47426fa7787f24ccd7af7d. Latest consensus state: current_term: 1 leader_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:26.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.564903 28635 sys_catalog.cc:458] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf [sys.catalog]: This master's current role is: FOLLOWER
13:39:26.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:26.587041 28651 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
13:39:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.920058 28633 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.920702 28633 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.921455 28633 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:26.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.973932 28633 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:26.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:26.975287 28633 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.176.1
13:39:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.033413 28633 tablet_server_runner.cc:78] Tablet server non-default flags:
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/wal
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.1:0
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data/info.pb
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.1
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:27.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.176.1
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/logs
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:39:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.034927 28633 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.037128 28633 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:27.073128 28657 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.115016 28595 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a03772693929407ab969350af3b6f461" member_type: VOTER last_known_addr { host: "127.27.176.62" port: 40085 } } peers { permanent_uuid: "00dd22677867449f844dca31954804bf" member_type: VOTER last_known_addr { host: "127.27.176.61" port: 41509 } } peers { permanent_uuid: "bdfb3438cc47426fa7787f24ccd7af7d" member_type: VOTER last_known_addr { host: "127.27.176.60" port: 45615 } } }
13:39:27.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.116164 28595 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdfb3438cc47426fa7787f24ccd7af7d [sys.catalog]: This master's current role is: LEADER
13:39:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.201058 28624 catalog_manager.cc:1259] Loaded cluster ID: 39db064ae6df4c52ae243a0d3f56fc85
13:39:27.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.201792 28624 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: loading cluster ID for follower catalog manager: success
13:39:27.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.210085 28624 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: acquiring CA information for follower catalog manager: success
13:39:27.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.222257 28624 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a03772693929407ab969350af3b6f461: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:39:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.617626 28651 catalog_manager.cc:1259] Loaded cluster ID: 39db064ae6df4c52ae243a0d3f56fc85
13:39:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.624562 28651 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf: loading cluster ID for follower catalog manager: success
13:39:27.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.641235 28651 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf: acquiring CA information for follower catalog manager: success
13:39:27.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:27.650327 28651 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 00dd22677867449f844dca31954804bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:39:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:27.075459 28658 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:28.384627 28633 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.312s user 0.433s sys 0.679s
13:39:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:28.385042 28633 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.313s user 0.434s sys 0.679s
13:39:28.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.389145 28633 server_base.cc:1034] running on GCE node
13:39:28.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:28.391647 28666 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:28.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.393225 28633 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:28.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.398989 28633 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:28.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.400539 28633 hybrid_clock.cc:648] HybridClock initialized: now 1714138768400495 us; error 42 us; skew 500 ppm
13:39:28.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.401818 28633 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:28.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.457427 28633 webserver.cc:460] Webserver started at http://127.27.176.1:39153/ using document root <none> and password file <none>
13:39:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.459048 28633 fs_manager.cc:359] Metadata directory not provided
13:39:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.459626 28633 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:28.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.460355 28633 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:28.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.467404 28633 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data/instance:
13:39:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0c040aa5ae445f6b675995bcc4e146c"
13:39:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:28 on dist-test-slave-d293"
13:39:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.469640 28633 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/wal/instance:
13:39:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0c040aa5ae445f6b675995bcc4e146c"
13:39:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:28 on dist-test-slave-d293"
13:39:28.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.484627 28633 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.004s sys 0.009s
13:39:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.498661 28671 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:28.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.500348 28633 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.003s sys 0.006s
13:39:28.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.500972 28633 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/wal
13:39:28.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0c040aa5ae445f6b675995bcc4e146c"
13:39:28.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:28 on dist-test-slave-d293"
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.501734 28633 fs_report.cc:383] FS layout report
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/wal
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/wal
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data/data
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:28.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.563732 28633 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.565745 28633 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.566341 28633 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:28.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.569845 28633 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:39:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.575966 28633 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:39:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.576246 28633 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:39:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.576573 28633 ts_tablet_manager.cc:590] Registered 0 tablets
13:39:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.576800 28633 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:39:28.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.776752 28633 rpc_server.cc:307] RPC server started. Bound to: 127.27.176.1:41855
13:39:28.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.776888 28784 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.176.1:41855 every 8 connection(s)
13:39:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.780761 28633 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/data/info.pb
13:39:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.790127 28352 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 28633
13:39:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.790570 28352 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-0/wal/instance
13:39:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.800318 28352 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/wal
13:39:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data
13:39:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/logs
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data/info.pb
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:28.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.2:0
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.176.2
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.2
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:39:28.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.869506 28785 heartbeater.cc:347] Connected to a master server at 127.27.176.60:45615
13:39:28.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.869975 28785 heartbeater.cc:464] Registering TS with master...
13:39:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.871170 28785 heartbeater.cc:511] Master 127.27.176.60:45615 requested a full tablet report, sending...
13:39:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.871989 28786 heartbeater.cc:347] Connected to a master server at 127.27.176.61:41509
13:39:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.872309 28786 heartbeater.cc:464] Registering TS with master...
13:39:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.873168 28786 heartbeater.cc:511] Master 127.27.176.61:41509 requested a full tablet report, sending...
13:39:28.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.874132 28551 ts_manager.cc:194] Registered new tserver with Master: a0c040aa5ae445f6b675995bcc4e146c (127.27.176.1:41855)
13:39:28.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.876672 28551 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.176.1:35719
13:39:28.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.877074 28472 ts_manager.cc:194] Registered new tserver with Master: a0c040aa5ae445f6b675995bcc4e146c (127.27.176.1:41855)
13:39:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.883603 28790 heartbeater.cc:347] Connected to a master server at 127.27.176.62:40085
13:39:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.883945 28790 heartbeater.cc:464] Registering TS with master...
13:39:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.884621 28790 heartbeater.cc:511] Master 127.27.176.62:40085 requested a full tablet report, sending...
13:39:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:28.887435 28397 ts_manager.cc:194] Registered new tserver with Master: a0c040aa5ae445f6b675995bcc4e146c (127.27.176.1:41855)
13:39:29.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.284327 28789 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.284977 28789 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.285722 28789 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.338840 28789 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:29.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.340198 28789 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.176.2
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:29.397641 28789 tablet_server_runner.cc:78] Tablet server non-default flags:
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/wal
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.2:0
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data/info.pb
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.2
13:39:29.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.176.2
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/logs
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:29.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:29.399128 28789 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:29.401126 28789 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.423524 28801 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:29.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:29.880715 28785 heartbeater.cc:503] Master 127.27.176.60:45615 was elected leader, sending a full tablet report...
13:39:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.437495 28805 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:29.429082 28802 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:30.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:30.593176 28803 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1165 milliseconds
13:39:30.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.593322 28789 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:39:30.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.594625 28789 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:30.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.597306 28789 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.598886 28789 hybrid_clock.cc:648] HybridClock initialized: now 1714138770598850 us; error 34 us; skew 500 ppm
13:39:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.599925 28789 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:30.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.612766 28789 webserver.cc:460] Webserver started at http://127.27.176.2:35379/ using document root <none> and password file <none>
13:39:30.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.613611 28789 fs_manager.cc:359] Metadata directory not provided
13:39:30.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.613792 28789 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.614168 28789 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.618786 28789 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data/instance:
13:39:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94fd6f86280e4ca6b30b7a64030f1998"
13:39:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:30 on dist-test-slave-d293"
13:39:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.619771 28789 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/wal/instance:
13:39:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94fd6f86280e4ca6b30b7a64030f1998"
13:39:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:30 on dist-test-slave-d293"
13:39:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.626562 28789 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
13:39:30.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.631980 28811 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.632900 28789 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
13:39:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.633226 28789 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/wal
13:39:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94fd6f86280e4ca6b30b7a64030f1998"
13:39:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:30 on dist-test-slave-d293"
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.633543 28789 fs_report.cc:383] FS layout report
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/wal
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/wal
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data/data
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.683051 28789 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.684564 28789 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.685024 28789 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:30.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.687425 28789 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:39:30.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.691592 28789 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:39:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.691793 28789 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:39:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.692073 28789 ts_tablet_manager.cc:590] Registered 0 tablets
13:39:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.692246 28789 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:39:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.827392 28789 rpc_server.cc:307] RPC server started. Bound to: 127.27.176.2:45403
13:39:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.827484 28924 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.176.2:45403 every 8 connection(s)
13:39:30.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.829903 28789 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/data/info.pb
13:39:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.836474 28352 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 28789
13:39:30.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.837127 28352 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-1/wal/instance
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.851658 28352 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/wal
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/logs
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data/info.pb
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.3:0
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.176.3
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.3
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:39:30.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.878468 28927 heartbeater.cc:347] Connected to a master server at 127.27.176.62:40085
13:39:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.878865 28927 heartbeater.cc:464] Registering TS with master...
13:39:30.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.879077 28926 heartbeater.cc:347] Connected to a master server at 127.27.176.61:41509
13:39:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.879443 28925 heartbeater.cc:347] Connected to a master server at 127.27.176.60:45615
13:39:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.879525 28926 heartbeater.cc:464] Registering TS with master...
13:39:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.879823 28925 heartbeater.cc:464] Registering TS with master...
13:39:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.880285 28927 heartbeater.cc:511] Master 127.27.176.62:40085 requested a full tablet report, sending...
13:39:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.880651 28926 heartbeater.cc:511] Master 127.27.176.61:41509 requested a full tablet report, sending...
13:39:30.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.880688 28925 heartbeater.cc:511] Master 127.27.176.60:45615 requested a full tablet report, sending...
13:39:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.883133 28472 ts_manager.cc:194] Registered new tserver with Master: 94fd6f86280e4ca6b30b7a64030f1998 (127.27.176.2:45403)
13:39:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.883247 28397 ts_manager.cc:194] Registered new tserver with Master: 94fd6f86280e4ca6b30b7a64030f1998 (127.27.176.2:45403)
13:39:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.883373 28551 ts_manager.cc:194] Registered new tserver with Master: 94fd6f86280e4ca6b30b7a64030f1998 (127.27.176.2:45403)
13:39:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:30.884963 28551 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.176.2:49523
13:39:31.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.154544 28933 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:31.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.155047 28933 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:31.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.155617 28933 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.184275 28933 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.185209 28933 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.176.3
13:39:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:31.219488 28933 tablet_server_runner.cc:78] Tablet server non-default flags:
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.176.20:40631
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/wal
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.176.3:0
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data/info.pb
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.176.3
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.176.62:40085,127.27.176.61:41509,127.27.176.60:45615
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.176.3
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/logs
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:31.220973 28933 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:31.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:31.222559 28933 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:31.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.239658 28943 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:31.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:31.888033 28925 heartbeater.cc:503] Master 127.27.176.60:45615 was elected leader, sending a full tablet report...
13:39:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.239675 28942 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:31.239828 28945 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:32.286414 28944 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1045 milliseconds
13:39:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.286533 28933 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:39:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.287612 28933 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.289937 28933 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.291265 28933 hybrid_clock.cc:648] HybridClock initialized: now 1714138772291238 us; error 43 us; skew 500 ppm
13:39:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.291986 28933 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:32.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.298111 28933 webserver.cc:460] Webserver started at http://127.27.176.3:39873/ using document root <none> and password file <none>
13:39:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.298952 28933 fs_manager.cc:359] Metadata directory not provided
13:39:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.299137 28933 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.299535 28933 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.304402 28933 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data/instance:
13:39:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24cca75ca1b4721b56825d3109fe94d"
13:39:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:32 on dist-test-slave-d293"
13:39:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.305473 28933 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/wal/instance:
13:39:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24cca75ca1b4721b56825d3109fe94d"
13:39:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:32 on dist-test-slave-d293"
13:39:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.311916 28933 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
13:39:32.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.316996 28952 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.317890 28933 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
13:39:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.318203 28933 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/wal
13:39:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24cca75ca1b4721b56825d3109fe94d"
13:39:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:32 on dist-test-slave-d293"
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.318552 28933 fs_report.cc:383] FS layout report
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/wal
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/wal
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data/data
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:32.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.369577 28933 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.371075 28933 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.371517 28933 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:32.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.373939 28933 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:39:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.377848 28933 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:39:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.378051 28933 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:39:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.378332 28933 ts_tablet_manager.cc:590] Registered 0 tablets
13:39:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.378497 28933 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:39:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.512810 28933 rpc_server.cc:307] RPC server started. Bound to: 127.27.176.3:41161
13:39:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.512910 29064 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.176.3:41161 every 8 connection(s)
13:39:32.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.515326 28933 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/data/info.pb
13:39:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.524384 28352 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 28933
13:39:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.524988 28352 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster911589755932882025/ts-2/wal/instance
13:39:32.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.545493 29066 heartbeater.cc:347] Connected to a master server at 127.27.176.61:41509
13:39:32.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.545897 29066 heartbeater.cc:464] Registering TS with master...
13:39:32.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.546953 29066 heartbeater.cc:511] Master 127.27.176.61:41509 requested a full tablet report, sending...
13:39:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.549374 29065 heartbeater.cc:347] Connected to a master server at 127.27.176.60:45615
13:39:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.549665 29065 heartbeater.cc:464] Registering TS with master...
13:39:32.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.549819 28472 ts_manager.cc:194] Registered new tserver with Master: f24cca75ca1b4721b56825d3109fe94d (127.27.176.3:41161)
13:39:32.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.550419 29065 heartbeater.cc:511] Master 127.27.176.60:45615 requested a full tablet report, sending...
13:39:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.551219 29067 heartbeater.cc:347] Connected to a master server at 127.27.176.62:40085
13:39:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.551496 29067 heartbeater.cc:464] Registering TS with master...
13:39:32.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.552278 29067 heartbeater.cc:511] Master 127.27.176.62:40085 requested a full tablet report, sending...
13:39:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.553064 28551 ts_manager.cc:194] Registered new tserver with Master: f24cca75ca1b4721b56825d3109fe94d (127.27.176.3:41161)
13:39:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.554625 28551 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.176.3:57909
13:39:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.554723 28397 ts_manager.cc:194] Registered new tserver with Master: f24cca75ca1b4721b56825d3109fe94d (127.27.176.3:41161)
13:39:32.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:32.562273 28352 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
13:39:32.572 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:39:32.573 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
13:39:32.584 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.176.62"
port: 40085
}
bound_http_address {
host: "127.27.176.62"
port: 40051
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.176.61"
port: 41509
}
bound_http_address {
host: "127.27.176.61"
port: 42895
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.176.60"
port: 45615
}
bound_http_address {
host: "127.27.176.60"
port: 43619
}
}
}
13:39:32.590 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:39:32.593 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.176.1"
port: 41855
}
bound_http_address {
host: "127.27.176.1"
port: 39153
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.176.2"
port: 45403
}
bound_http_address {
host: "127.27.176.2"
port: 35379
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.176.3"
port: 41161
}
bound_http_address {
host: "127.27.176.3"
port: 39873
}
}
}
13:39:32.593 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
13:39:32.759 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
13:39:32.881 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskPWbCo1/test-tmp
13:39:32.882 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:39:32.882 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskPWbCo1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:39:32.885 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
13:39:33.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:33.208 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:39:33.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:33.212781 29077 mini_oidc.cc:152] Starting JWKS server
13:39:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:33.216066 29077 webserver.cc:460] Webserver started at http://127.0.0.1:32777/ using document root <none> and password file <none>
13:39:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:33.216480 29077 mini_oidc.cc:189] Starting OIDC Discovery server
13:39:33.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:33.218112 29077 webserver.cc:460] Webserver started at http://127.0.0.1:33523/ using document root <none> and password file <none>
13:39:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
13:39:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/principal' for realm 'KRBTEST.COM',
13:39:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
13:39:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:33 dist-test-slave-d293 krb5kdc[29089](info): setting up network...
13:39:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
13:39:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:33 dist-test-slave-d293 krb5kdc[29089](info): set up 2 sockets
13:39:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:33 dist-test-slave-d293 krb5kdc[29089](info): commencing operation
13:39:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
13:39:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:33.557668 29065 heartbeater.cc:503] Master 127.27.176.60:45615 was elected leader, sending a full tablet report...
13:39:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.169929 29077 mini_kdc.cc:121] Time spent starting KDC: real 0.917s user 0.002s sys 0.014s
13:39:34.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
13:39:34.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:39:34.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
13:39:34.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
13:39:34.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:39:34.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
13:39:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
13:39:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:39:34.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
13:39:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:39:34.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/test-user.keytab.
13:39:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/test-user.keytab.
13:39:34.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:34 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:34.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:39:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:39:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:39:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:39:34Z Disabled control of system clock
13:39:34.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.101.126@KRBTEST.COM; defaulting to no policy
13:39:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:39:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.101.126@KRBTEST.COM" created.
13:39:34.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:39:34.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.101.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.126.keytab.
13:39:34.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.101.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.126.keytab.
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:34.332890 29077 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/logs
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/info.pb
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.126:34059
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.126
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.126.keytab
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.126
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:39:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:39:34.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.636186 29103 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:34.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.636850 29103 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:34.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.637547 29103 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.667512 29103 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:39:34.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.667753 29103 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.668118 29103 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.668414 29103 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.668766 29103 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.669073 29103 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.669312 29103 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32777/jwks.json
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:34.704059 29103 master_runner.cc:386] Master server non-default flags:
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.126.keytab
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.126
13:39:34.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.126:34059
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/info.pb
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.126
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/logs
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:34.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:34.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:34.705466 29103 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:34.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:34.706570 29103 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.721318 29110 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.721374 29109 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:34.721379 29112 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:34.722843 29103 server_base.cc:1034] running on GCE node
13:39:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.762388 29103 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.764990 29103 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:35.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.766319 29103 hybrid_clock.cc:648] HybridClock initialized: now 1714138775766293 us; error 47 us; skew 500 ppm
13:39:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.767035 29103 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:35 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138775, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.778107 29103 init.cc:377] Logged in from keytab as kudu/127.28.101.126@KRBTEST.COM (short username kudu)
13:39:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.783915 29103 webserver.cc:460] Webserver started at http://127.28.101.126:43991/ using document root <none> and password file <none>
13:39:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.786068 29103 fs_manager.cc:359] Metadata directory not provided
13:39:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.786782 29103 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.787278 29103 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:35.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.791972 29103 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/instance:
13:39:35.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb24cc31e50c43719299787971659ccf"
13:39:35.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:35 on dist-test-slave-d293"
13:39:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.793009 29103 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal/instance:
13:39:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb24cc31e50c43719299787971659ccf"
13:39:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:35 on dist-test-slave-d293"
13:39:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.799638 29103 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
13:39:35.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.805099 29120 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:35.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.806345 29103 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.806732 29103 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb24cc31e50c43719299787971659ccf"
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:35 on dist-test-slave-d293"
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.807142 29103 fs_report.cc:383] FS layout report
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/data
13:39:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:35.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.859354 29103 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:35.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.864724 29103 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:35.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.865106 29103 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.937040 29103 rpc_server.cc:307] RPC server started. Bound to: 127.28.101.126:34059
13:39:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.937211 29171 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.101.126:34059 every 8 connection(s)
13:39:35.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.939599 29103 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/info.pb
13:39:35.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.944941 29172 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:39:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.946513 29077 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 29103
13:39:35.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.947037 29077 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal/instance
13:39:35.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.958845 29172 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } has no permanent_uuid. Determining permanent_uuid...
13:39:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.101.125@KRBTEST.COM; defaulting to no policy
13:39:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:39:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.101.125@KRBTEST.COM" created.
13:39:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:35 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138775, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.126@KRBTEST.COM for kudu/127.28.101.126@KRBTEST.COM
13:39:35.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:39:35.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.101.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.125.keytab.
13:39:35.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.101.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.125.keytab.
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.988879 29077 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/logs
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/info.pb
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.125:40157
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.125
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.125.keytab
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.125
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:39:35.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:35.997545 29172 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } has no permanent_uuid. Determining permanent_uuid...
13:39:36.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.000416 29124 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.005187 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.125:40157: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:36.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.054690 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } attempt: 1
13:39:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.058504 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.125:40157: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:36.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.126875 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } attempt: 2
13:39:36.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.130497 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.125:40157: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:36.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.221846 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } attempt: 3
13:39:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.225446 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.125:40157: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.273766 29178 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.274252 29178 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.274819 29178 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.303357 29178 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:39:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.303647 29178 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:36.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.304001 29178 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:36.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.304247 29178 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:36.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.304477 29178 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:36.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.304685 29178 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:36.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.304873 29178 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32777/jwks.json
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.338994 29178 master_runner.cc:386] Master server non-default flags:
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.125.keytab
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.125
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.125:40157
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/info.pb
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.125
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/logs
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.340096 29178 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:36.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.341238 29178 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:36.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.354456 29186 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.368803 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } attempt: 4
13:39:36.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.373379 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.125:40157: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:36.672816 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } attempt: 5
13:39:36.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.677855 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.125:40157: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.225293 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } attempt: 6
13:39:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.234897 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.125:40157: Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:37.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.357975 29188 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:36.354463 29185 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.421932 29187 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1065 milliseconds
13:39:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.422091 29178 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:39:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.423287 29178 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.425803 29178 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.427156 29178 hybrid_clock.cc:648] HybridClock initialized: now 1714138777427114 us; error 48 us; skew 500 ppm
13:39:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.428042 29178 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:37 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138777, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.433691 29178 init.cc:377] Logged in from keytab as kudu/127.28.101.125@KRBTEST.COM (short username kudu)
13:39:37.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.439363 29178 webserver.cc:460] Webserver started at http://127.28.101.125:46175/ using document root <none> and password file <none>
13:39:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.440469 29178 fs_manager.cc:359] Metadata directory not provided
13:39:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.440739 29178 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.441272 29178 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.446848 29178 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/instance:
13:39:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba0b8e3596164dbcaf584f0068c4da01"
13:39:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:37 on dist-test-slave-d293"
13:39:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.447928 29178 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal/instance:
13:39:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba0b8e3596164dbcaf584f0068c4da01"
13:39:37.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:37 on dist-test-slave-d293"
13:39:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.454954 29178 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
13:39:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.459836 29199 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.460753 29178 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
13:39:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.461084 29178 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba0b8e3596164dbcaf584f0068c4da01"
13:39:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:37 on dist-test-slave-d293"
13:39:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.461392 29178 fs_report.cc:383] FS layout report
13:39:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/data
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.504241 29178 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.509323 29178 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.509785 29178 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:37.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.578696 29178 rpc_server.cc:307] RPC server started. Bound to: 127.28.101.125:40157
13:39:37.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.578771 29250 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.101.125:40157 every 8 connection(s)
13:39:37.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.582345 29178 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/info.pb
13:39:37.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.583527 29077 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 29178
13:39:37.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.584014 29077 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal/instance
13:39:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.588577 29251 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:39:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.101.124@KRBTEST.COM; defaulting to no policy
13:39:37.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:39:37.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.101.124@KRBTEST.COM" created.
13:39:37.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.604298 29251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } has no permanent_uuid. Determining permanent_uuid...
13:39:37.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:37 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138777, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.125@KRBTEST.COM for kudu/127.28.101.126@KRBTEST.COM
13:39:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:39:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.101.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.124.keytab.
13:39:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.101.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.124.keytab.
13:39:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.628363 29077 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data
13:39:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/logs
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/info.pb
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.124:36159
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.124
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.124.keytab
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.124
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:37.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:39:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.634728 29251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } has no permanent_uuid. Determining permanent_uuid...
13:39:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:37 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138777, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.125@KRBTEST.COM for kudu/127.28.101.125@KRBTEST.COM
13:39:37.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.659024 29251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } has no permanent_uuid. Determining permanent_uuid...
13:39:37.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.661242 29202 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:37.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.665028 29251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:37.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.714438 29251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 1
13:39:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.718283 29251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.786620 29251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 2
13:39:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.790692 29251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:37.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.882113 29251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 3
13:39:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.885540 29251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.928388 29257 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.928938 29257 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.929553 29257 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.958968 29257 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:39:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.959160 29257 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.959416 29257 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.959610 29257 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:37.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.959798 29257 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.959988 29257 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:37.960127 29257 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32777/jwks.json
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.993294 29257 master_runner.cc:386] Master server non-default flags:
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.124.keytab
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.124
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.124:36159
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/info.pb
13:39:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.124
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/logs
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.994315 29257 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:37.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:37.995241 29257 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.009946 29267 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.028930 29251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 4
13:39:38.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.033207 29251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.295378 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } attempt: 7
13:39:38.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:38 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138775, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.126@KRBTEST.COM for kudu/127.28.101.125@KRBTEST.COM
13:39:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.332652 29251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 5
13:39:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.338057 29251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:38.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.339536 29172 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } has no permanent_uuid. Determining permanent_uuid...
13:39:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.346007 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.404497 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 1
13:39:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.409505 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:38.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.490962 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 2
13:39:38.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.496307 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.581789 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 3
13:39:38.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.591743 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:38.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.732156 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 4
13:39:38.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.737568 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:38.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:38.885485 29251 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 6
13:39:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.893059 29251 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.021010 29172 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } attempt: 5
13:39:39.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.026744 29172 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.28.101.124:36159: Network error: Client connection negotiation failed: client connection to 127.28.101.124:36159: connect: Connection refused (error 111)
13:39:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.009975 29266 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:38.012534 29269 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:39.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.124748 29268 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1112 milliseconds
13:39:39.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.124850 29257 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:39:39.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.125981 29257 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.128398 29257 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.129783 29257 hybrid_clock.cc:648] HybridClock initialized: now 1714138779129755 us; error 37 us; skew 500 ppm
13:39:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.130501 29257 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:39.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138779, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.135826 29257 init.cc:377] Logged in from keytab as kudu/127.28.101.124@KRBTEST.COM (short username kudu)
13:39:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.140542 29257 webserver.cc:460] Webserver started at http://127.28.101.124:43515/ using document root <none> and password file <none>
13:39:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.141366 29257 fs_manager.cc:359] Metadata directory not provided
13:39:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.141551 29257 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.141973 29257 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:39:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.146631 29257 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/instance:
13:39:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3009d4d333f04d87a7241803caf4c241"
13:39:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:39 on dist-test-slave-d293"
13:39:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.147634 29257 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal/instance:
13:39:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3009d4d333f04d87a7241803caf4c241"
13:39:39.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:39 on dist-test-slave-d293"
13:39:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.154407 29257 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.003s
13:39:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.159353 29282 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.160418 29257 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.160740 29257 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3009d4d333f04d87a7241803caf4c241"
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:39 on dist-test-slave-d293"
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.161020 29257 fs_report.cc:383] FS layout report
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/data
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:39.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.209429 29257 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.214365 29257 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.214753 29257 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.283437 29257 rpc_server.cc:307] RPC server started. Bound to: 127.28.101.124:36159
13:39:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.283509 29334 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.101.124:36159 every 8 connection(s)
13:39:39.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.285858 29257 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/info.pb
13:39:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.291045 29335 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:39:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.295150 29077 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 29257
13:39:39.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.295615 29077 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal/instance
13:39:39.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.304129 29335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } has no permanent_uuid. Determining permanent_uuid...
13:39:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138779, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for kudu/127.28.101.126@KRBTEST.COM
13:39:39.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.101.124@KRBTEST.COM
13:39:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.336269 29335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } has no permanent_uuid. Determining permanent_uuid...
13:39:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138779, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for kudu/127.28.101.125@KRBTEST.COM
13:39:39.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.101.125@KRBTEST.COM
13:39:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.362413 29335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } has no permanent_uuid. Determining permanent_uuid...
13:39:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138779, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for kudu/127.28.101.124@KRBTEST.COM
13:39:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.101.126@KRBTEST.COM
13:39:39.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.387897 29335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Bootstrap starting.
13:39:39.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.391810 29077 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
13:39:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.394171 29335 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Neither blocks nor log segments found. Creating new log.
13:39:39.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.396194 29335 log.cc:824] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Log is configured to *not* fsync() on all Append() calls
13:39:39.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:39.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.400257 29335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: No bootstrap required, opened a new log
13:39:39.401 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
13:39:39.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
13:39:39.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
13:39:39.415 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.101.126"
port: 34059
}
bound_http_address {
host: "127.28.101.126"
port: 43991
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.101.125"
port: 40157
}
bound_http_address {
host: "127.28.101.125"
port: 46175
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.101.124"
port: 36159
}
bound_http_address {
host: "127.28.101.124"
port: 43515
}
}
}
13:39:39.415 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:39:39.416 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:39.416 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:39:39.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:39:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.424316 29335 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:39.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.424959 29335 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:39:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.425228 29335 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3009d4d333f04d87a7241803caf4c241, State: Initialized, Role: FOLLOWER
13:39:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.426003 29335 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:39.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.428894 29345 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } } }
13:39:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.429940 29345 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [sys.catalog]: This master's current role is: FOLLOWER
13:39:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.430611 29335 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [sys.catalog]: configured and running, proceeding with master startup.
13:39:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:39:39.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:39.456 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc
13:39:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.457692 29357 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.458006 29357 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:39.463 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:39:39.464 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc
13:39:39.465 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:39:39.469 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
13:39:39.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.469708 29288 tablet.cc:2369] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:39:39.471 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:39:39.471 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:39:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.454310 29257 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: OK, instance_id { permanent_uuid: "bb24cc31e50c43719299787971659ccf" instance_seqno: 1714138775918409 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:39:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.485612 29257 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:39:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:39 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:39:39.500 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:39.500 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc
13:39:39.502 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:39:39.502 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.101.125:40157
13:39:39.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
13:39:39.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.505286 29077 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 29178
13:39:39.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:39.528 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.101.124:36159
13:39:39.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
13:39:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.529522 29077 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 29257
13:39:39.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:39.546 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.101.126:34059
13:39:39.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
13:39:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.547294 29077 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 29103
13:39:39.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:39.564 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.101.125:40157
13:39:39.566 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
13:39:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.567561 29077 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data
13:39:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/logs
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/info.pb
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.125:40157
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.125
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46175
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.125.keytab
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.125
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:39.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:39.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:39:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.885337 29361 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.885850 29361 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:39.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.886405 29361 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.916793 29361 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:39:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.917003 29361 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.917261 29361 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.917465 29361 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.917652 29361 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:39.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.917836 29361 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:39.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.917979 29361 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32777/jwks.json
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.955260 29361 master_runner.cc:386] Master server non-default flags:
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:39.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.125.keytab
13:39:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.125
13:39:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.125:40157
13:39:39.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/info.pb
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.125
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46175
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/logs
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:39.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.956410 29361 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:39.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.957968 29361 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:39.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.974131 29370 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:41.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.974287 29368 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:41.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:39.974694 29367 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:41.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:39.975867 29361 server_base.cc:1034] running on GCE node
13:39:41.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.022218 29361 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:41.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.025092 29361 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.026525 29361 hybrid_clock.cc:648] HybridClock initialized: now 1714138781026494 us; error 48 us; skew 500 ppm
13:39:41.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.027295 29361 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:41 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138781, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:41.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.036597 29361 init.cc:377] Logged in from keytab as kudu/127.28.101.125@KRBTEST.COM (short username kudu)
13:39:41.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.041738 29361 webserver.cc:460] Webserver started at http://127.28.101.125:46175/ using document root <none> and password file <none>
13:39:41.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.042572 29361 fs_manager.cc:359] Metadata directory not provided
13:39:41.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.042758 29361 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:41.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.050185 29361 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
13:39:41.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.054111 29380 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:41.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.055058 29361 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
13:39:41.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.055351 29361 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:41.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba0b8e3596164dbcaf584f0068c4da01"
13:39:41.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:37 on dist-test-slave-d293"
13:39:41.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.057570 29361 fs_report.cc:383] FS layout report
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/data
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:41.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.098577 29361 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.103962 29361 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.104502 29361 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.175051 29361 rpc_server.cc:307] RPC server started. Bound to: 127.28.101.125:40157
13:39:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.175168 29433 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.101.125:40157 every 8 connection(s)
13:39:41.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.177785 29361 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/data/info.pb
13:39:41.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.181094 29077 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 29361
13:39:41.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:41.181 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.101.124:36159
13:39:41.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.183048 29077 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/logs
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/info.pb
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:41.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.124:36159
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.124
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43515
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.124.keytab
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.124
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:41.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:39:41.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.188261 29434 sys_catalog.cc:263] Verifying existing consensus state
13:39:41.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 13:39:41.189119 29434 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:39:41.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.189590 29361 master.cc:537] Master@127.28.101.125:40157 shutting down...
13:39:41.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.257094 29361 master.cc:552] Master@127.28.101.125:40157 shutdown complete.
13:39:41.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:39:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.493391 29436 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.493892 29436 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:41.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.494488 29436 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:41.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.523638 29436 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:39:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.523847 29436 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.524117 29436 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.524312 29436 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.524544 29436 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.524741 29436 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:41.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.524888 29436 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32777/jwks.json
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.559717 29436 master_runner.cc:386] Master server non-default flags:
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.124.keytab
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.124
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.124:36159
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/info.pb
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.124
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43515
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/logs
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.561029 29436 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:41.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.562325 29436 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:41.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.577111 29443 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.577136 29445 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:41.578053 29442 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:41.578814 29436 server_base.cc:1034] running on GCE node
13:39:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.636195 29436 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:42.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.639315 29436 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.640863 29436 hybrid_clock.cc:648] HybridClock initialized: now 1714138782640826 us; error 53 us; skew 500 ppm
13:39:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.641661 29436 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:42.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:42 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138782, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:42.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.647400 29436 init.cc:377] Logged in from keytab as kudu/127.28.101.124@KRBTEST.COM (short username kudu)
13:39:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.651983 29436 webserver.cc:460] Webserver started at http://127.28.101.124:43515/ using document root <none> and password file <none>
13:39:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.652879 29436 fs_manager.cc:359] Metadata directory not provided
13:39:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.653105 29436 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:42.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.660423 29436 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.002s
13:39:42.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.664691 29453 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.665601 29436 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.665911 29436 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3009d4d333f04d87a7241803caf4c241"
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:39 on dist-test-slave-d293"
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.668051 29436 fs_report.cc:383] FS layout report
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/wal
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/data
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.710592 29436 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.715433 29436 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.715883 29436 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:42.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.783865 29436 rpc_server.cc:307] RPC server started. Bound to: 127.28.101.124:36159
13:39:42.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.783955 29505 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.101.124:36159 every 8 connection(s)
13:39:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.787420 29436 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-2/data/info.pb
13:39:42.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.793893 29077 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 29436
13:39:42.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:42.795 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.101.126:34059
13:39:42.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
13:39:42.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.796626 29077 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/logs
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/info.pb
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.126:34059
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.126
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43991
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.126.keytab
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.126
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:39:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.797664 29506 sys_catalog.cc:263] Verifying existing consensus state
13:39:42.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.803789 29506 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Bootstrap starting.
13:39:42.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.846204 29506 log.cc:824] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Log is configured to *not* fsync() on all Append() calls
13:39:42.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.851205 29506 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:39:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.852059 29506 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Bootstrap complete.
13:39:42.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.885916 29506 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:42.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.889216 29506 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:39:42.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.889528 29506 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3009d4d333f04d87a7241803caf4c241, State: Initialized, Role: FOLLOWER
13:39:42.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.890460 29506 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.893102 29510 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } } }
13:39:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.893884 29510 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [sys.catalog]: This master's current role is: FOLLOWER
13:39:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.894893 29506 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [sys.catalog]: configured and running, proceeding with master startup.
13:39:42.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:42.921180 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:42.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:42.921975 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.927240 29436 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111),
13:39:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:42.963207 29436 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:39:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:42.970606 29459 tablet.cc:2369] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:39:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.118508 29510 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:39:43.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.119099 29510 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:43.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.122192 29510 leader_election.cc:288] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bb24cc31e50c43719299787971659ccf (127.28.101.126:34059), ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157)
13:39:43.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.125530 29456 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:39:43.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.141113 29456 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb24cc31e50c43719299787971659ccf (127.28.101.126:34059): Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:39:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.142082 29457 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157): Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.142516 29457 leader_election.cc:302] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3009d4d333f04d87a7241803caf4c241; no voters: ba0b8e3596164dbcaf584f0068c4da01, bb24cc31e50c43719299787971659ccf
13:39:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.143343 29510 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:39:43.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.159281 29508 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:39:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.160094 29508 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:39:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.160863 29508 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:39:43.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.191275 29508 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:39:43.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.191526 29508 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:39:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.191902 29508 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:39:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.192198 29508 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:39:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.192507 29508 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:39:43.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.192804 29508 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:39:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.193011 29508 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32777/jwks.json
13:39:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.227878 29508 master_runner.cc:386] Master server non-default flags:
13:39:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:39:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.101.84:36211
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.101.126:34059,127.28.101.125:40157,127.28.101.124:36159
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/krb5kdc/kudu_127.28.101.126.keytab
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.101.126
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:39:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.101.126:34059
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32777/jwks.json
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/info.pb
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.101.126
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43991
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/logs
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 966ea5b322fe1ecae0e32a71097d04728a5f9d6f
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Apr 2024 13:00:19 UTC on 1a15035a6830
13:39:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 265
13:39:43.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:39:43.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.229377 29508 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:39:43.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.230722 29508 file_cache.cc:492] Constructed file cache file cache with capacity 419430
13:39:43.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.246544 29535 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:43.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.926352 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:43.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.926707 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:44.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:43.248469 29508 server_base.cc:1034] running on GCE node
13:39:44.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.253176 29532 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:44.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:43.247104 29533 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:39:44.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.369479 29508 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:39:44.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.372748 29508 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:39:44.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.374197 29508 hybrid_clock.cc:648] HybridClock initialized: now 1714138784374148 us; error 38 us; skew 500 ppm
13:39:44.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.375257 29508 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:39:44.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:44 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138784, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:44.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.381431 29508 init.cc:377] Logged in from keytab as kudu/127.28.101.126@KRBTEST.COM (short username kudu)
13:39:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.387265 29508 webserver.cc:460] Webserver started at http://127.28.101.126:43991/ using document root <none> and password file <none>
13:39:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.388554 29508 fs_manager.cc:359] Metadata directory not provided
13:39:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.388891 29508 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:39:44.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.400055 29508 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
13:39:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.405676 29545 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:39:44.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.406833 29508 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
13:39:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.407243 29508 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data,/tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bb24cc31e50c43719299787971659ccf"
13:39:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-04-26 13:39:35 on dist-test-slave-d293"
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.410601 29508 fs_report.cc:383] FS layout report
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/data
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:39:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:39:44.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.476099 29508 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
13:39:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.483657 29508 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:39:44.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.484218 29508 kserver.cc:163] Server-wide thread pool size limit: 3276
13:39:44.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.563571 29508 rpc_server.cc:307] RPC server started. Bound to: 127.28.101.126:34059
13:39:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.563781 29596 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.101.126:34059 every 8 connection(s)
13:39:44.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.567245 29508 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/data/info.pb
13:39:44.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.572981 29077 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu as pid 29508
13:39:44.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:39:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.580108 29597 sys_catalog.cc:263] Verifying existing consensus state
13:39:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240426 13:39:44.581041 29597 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:39:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.581538 29508 master.cc:537] Master@127.28.101.126:34059 shutting down...
13:39:44.631 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:44.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.661922 29508 master.cc:552] Master@127.28.101.126:34059 shutdown complete.
13:39:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskPWbCo1/test-tmp/mini-kudu-cluster2212214900155272206/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:39:44.695 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.101.124:36159(127.28.101.124:36159)] connecting to peer
13:39:44.705 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:44.735 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.101.124:36159(127.28.101.124:36159)] Successfully connected to peer
13:39:44.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:44.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:44.740 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:44.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:44.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:44.743 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:44.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.805382 29603 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:39:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.805836 29603 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.807735 29603 leader_election.cc:288] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bb24cc31e50c43719299787971659ccf (127.28.101.126:34059), ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157)
13:39:44.812 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
13:39:44.822 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.101.124
13:39:44.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:44.823205 29456 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb24cc31e50c43719299787971659ccf (127.28.101.126:34059): Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:39:44.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:44.838371 29457 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157): Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:44.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.838845 29457 leader_election.cc:302] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3009d4d333f04d87a7241803caf4c241; no voters: ba0b8e3596164dbcaf584f0068c4da01, bb24cc31e50c43719299787971659ccf
13:39:44.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:44.839530 29603 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:39:44.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:44.930915 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:44.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:44.931521 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:44 dist-test-slave-d293 krb5kdc[29089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.101.124@KRBTEST.COM
13:39:45.040 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2e5815b6, L:/127.0.0.1:35924 - R:/127.28.101.124:36159] using SASL/GSSAPI
13:39:45.054 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.060 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.064 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.064 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.065 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.067 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.067 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.068 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.071 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.079 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.080 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.082 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.082 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.083 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.085 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.099 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.101 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.103 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.103 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.103 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.104 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.104 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.106 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.119 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.120 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.122 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.122 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.123 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.123 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.123 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.124 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.125 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.159 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.161 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.164 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.164 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.164 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.164 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.167 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.168 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.199 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.202 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.204 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.204 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.204 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.205 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.206 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.207 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.239 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.241 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.242 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.242 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.242 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.243 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.244 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.245 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.439 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.441 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.443 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.443 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.444 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.444 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.444 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.446 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.447 [WARN - pool-6-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.779 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:45.780 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:45.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.782 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.782 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.782 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.782 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:45.782 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:45.785 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:45.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:45.935822 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:45.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:45.936061 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:46.499 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:46.501 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:46.502 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:46.502 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:46.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:46.502 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:46.502 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:46.502 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:46.504 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:46.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:46.610580 29502 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7bc0 after lost signal to thread 29437
13:39:46.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:46.611176 29502 debug-util.cc:398] Leaking SignalData structure 0x7b0800099d80 after lost signal to thread 29505
13:39:46.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:46.939177 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:46.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:46.939500 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:46.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:46.952184 29608 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:39:46.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:46.952572 29608 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:46.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:46.953874 29608 leader_election.cc:288] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bb24cc31e50c43719299787971659ccf (127.28.101.126:34059), ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157)
13:39:46.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:46.959025 29456 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb24cc31e50c43719299787971659ccf (127.28.101.126:34059): Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:39:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:46.961017 29457 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157): Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:46.961552 29457 leader_election.cc:302] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3009d4d333f04d87a7241803caf4c241; no voters: ba0b8e3596164dbcaf584f0068c4da01, bb24cc31e50c43719299787971659ccf
13:39:46.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:46.962303 29608 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:39:47.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:47.942822 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:47.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:47.943140 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:48.004384 29436 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111),
13:39:48.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:48.004804 29436 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:39:48.199 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:48.200 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:48.202 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:48.202 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:48.202 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:48.202 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:48.202 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:48.202 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:48.203 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:48.818 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:48.820 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:48.821 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:48.822 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:48.822 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:48.822 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:48.822 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:48.822 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:48.824 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:48.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:48.946166 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:48.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:48.946432 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:49.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:49.054473 29613 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:39:49.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:49.054839 29613 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:49.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:49.056149 29613 leader_election.cc:288] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bb24cc31e50c43719299787971659ccf (127.28.101.126:34059), ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157)
13:39:49.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:49.057888 29456 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111) [suppressed 5 similar messages]
13:39:49.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:49.060683 29456 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb24cc31e50c43719299787971659ccf (127.28.101.126:34059): Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:39:49.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:49.061056 29457 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157): Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:49.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:49.061342 29457 leader_election.cc:302] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3009d4d333f04d87a7241803caf4c241; no voters: ba0b8e3596164dbcaf584f0068c4da01, bb24cc31e50c43719299787971659ccf
13:39:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:49.061834 29613 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:39:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:49.949970 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:49.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:49.950217 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:50.819 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:50.821 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:50.822 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:50.822 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:50.822 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:50.822 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:50.822 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:50.822 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:50.824 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:50.953672 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:50.954011 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:51.612943 29622 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:39:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:51.613535 29622 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:39:51.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:51.615216 29622 leader_election.cc:288] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bb24cc31e50c43719299787971659ccf (127.28.101.126:34059), ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157)
13:39:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:51.620467 29456 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb24cc31e50c43719299787971659ccf (127.28.101.126:34059): Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:39:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:51.620965 29457 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157): Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:39:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:51.621251 29457 leader_election.cc:302] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3009d4d333f04d87a7241803caf4c241; no voters: ba0b8e3596164dbcaf584f0068c4da01, bb24cc31e50c43719299787971659ccf
13:39:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:51.621781 29622 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:39:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:51.957814 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:51.958133 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:39:52 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138792, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:39:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:52.650856 29449 init.cc:295] Successfully reacquired a new kerberos TGT
13:39:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:52.961103 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:52.961352 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:53.007547 29436 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111),
13:39:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:53.007941 29436 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:39:53.859 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:53.860 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:53.862 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:53.862 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:53.862 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:53.863 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:53.863 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:53.863 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:53.864 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:53.964742 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:53.965052 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:54.579 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:54.581 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:54.584 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:54.584 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:54.584 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:54.584 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:54.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:54.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:54.586 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:54.967690 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:54.967960 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:55.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:55.970630 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:55.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:55.970875 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:56.499 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:56.500 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:56.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:56.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:56.502 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:56.503 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:56.503 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:56.503 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:56.504 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:56.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:56.974011 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:56.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:56.974280 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:57.899 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:39:57.901 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:39:57.902 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:57.903 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:57.903 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:39:57.903 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:57.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:57.904 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:39:57.905 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:39:57.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:57.977536 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:57.977790 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:58.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:58.010516 29436 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111),
13:39:58.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:39:58.010847 29436 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:39:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:58.980569 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:58.980825 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:39:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:59.983651 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:39:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:39:59.983943 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:00.339 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:40:00.341 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:40:00.342 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:00.343 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:00.343 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:00.344 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:00.344 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:00.344 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:00.345 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:40:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:00.986649 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:00.986919 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:01.162667 29634 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:40:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:01.163151 29634 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:40:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:01.164918 29634 leader_election.cc:288] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bb24cc31e50c43719299787971659ccf (127.28.101.126:34059), ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157)
13:40:01.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:01.166661 29456 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111) [suppressed 3 similar messages]
13:40:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:01.169304 29456 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb24cc31e50c43719299787971659ccf (127.28.101.126:34059): Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:40:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:01.169741 29457 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157): Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:40:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:01.170018 29457 leader_election.cc:302] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3009d4d333f04d87a7241803caf4c241; no voters: ba0b8e3596164dbcaf584f0068c4da01, bb24cc31e50c43719299787971659ccf
13:40:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:01.170572 29634 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:40:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:01.990482 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:01.990805 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:02.419 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:40:02.420 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:40:02.422 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:02.422 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:02.422 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:02.422 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:02.422 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:02.422 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:02.425 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:40:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:40:02 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138802, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:40:02.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:02.653115 29449 init.cc:295] Successfully reacquired a new kerberos TGT
13:40:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:02.993466 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:02.993731 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:03.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:03.016099 29436 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111),
13:40:03.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:03.016678 29436 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:40:03.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:03.996690 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:03.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:03.996949 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:04.439 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:40:04.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:04.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:04.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:04.441 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:40:04.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:04.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:04.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:04.445 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:40:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:04.999799 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:05.000062 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:06.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:06.002808 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:06.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:06.003072 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:06.139 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:40:06.140 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:40:06.141 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:06.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:06.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:06.141 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:06.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:06.142 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:06.144 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:40:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:07.005970 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:07.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:07.006229 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:08.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:08.008857 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:08.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:08.009119 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:08.019184 29436 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111),
13:40:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:08.019529 29436 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:40:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:09.011710 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:09.011974 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:09.979 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:40:09.980 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:40:09.981 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:09.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:09.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:09.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:09.982 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:09.982 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:09.984 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:40:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:10.014863 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:10.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:10.015120 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:11.018894 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:11.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:11.019156 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:11.039 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:40:11.040 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:40:11.042 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:11.042 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:11.042 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:11.043 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:11.043 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:11.043 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:11.044 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:40:11.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:11.622640 29652 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:40:11.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:11.623430 29652 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bb24cc31e50c43719299787971659ccf" member_type: VOTER last_known_addr { host: "127.28.101.126" port: 34059 } } peers { permanent_uuid: "ba0b8e3596164dbcaf584f0068c4da01" member_type: VOTER last_known_addr { host: "127.28.101.125" port: 40157 } } peers { permanent_uuid: "3009d4d333f04d87a7241803caf4c241" member_type: VOTER last_known_addr { host: "127.28.101.124" port: 36159 } }
13:40:11.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:11.625388 29652 leader_election.cc:288] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bb24cc31e50c43719299787971659ccf (127.28.101.126:34059), ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157)
13:40:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:11.627873 29456 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111) [suppressed 1 similar messages]
13:40:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:11.631383 29457 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ba0b8e3596164dbcaf584f0068c4da01 (127.28.101.125:40157): Network error: Client connection negotiation failed: client connection to 127.28.101.125:40157: connect: Connection refused (error 111)
13:40:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:11.631837 29456 leader_election.cc:334] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bb24cc31e50c43719299787971659ccf (127.28.101.126:34059): Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111)
13:40:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:11.632136 29456 leader_election.cc:302] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3009d4d333f04d87a7241803caf4c241; no voters: ba0b8e3596164dbcaf584f0068c4da01, bb24cc31e50c43719299787971659ccf
13:40:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:11.632669 29652 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:40:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:12.022415 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:12.022702 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Apr 26 13:40:12 dist-test-slave-d293 krb5kdc[29089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714138812, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.101.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:40:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:12.655231 29449 init.cc:295] Successfully reacquired a new kerberos TGT
13:40:13.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:13.025651 29521 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3009d4d333f04d87a7241803caf4c241: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:40:13.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:13.025545 29436 master_runner.cc:186] Error getting master registration for 127.28.101.126:34059: Network error: Client connection negotiation failed: client connection to 127.28.101.126:34059: connect: Connection refused (error 111),
13:40:13.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240426 13:40:13.025892 29521 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:40:13.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:13.026003 29436 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:40:13.679 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.125:40157(127.28.101.125:40157)] connecting to peer
13:40:13.680 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.101.126:34059(127.28.101.126:34059)] connecting to peer
13:40:13.681 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:13.681 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:13.682 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.101.126:34059(127.28.101.126:34059)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:13.682 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.125:40157(127.28.101.125:40157)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:13.682 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.126:34059: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059
13:40:13.682 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.101.125:40157: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
13:40:13.684 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059; will retry
13:40:13.689 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29107), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [153ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [156ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [465ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [468ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [473ms] refreshing cache from master, [474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [479ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [481ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [484ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [493ms] refreshing cache from master, [493ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [494ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [494ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [496ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [498ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [499ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [499ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [513ms] refreshing cache from master, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [517ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [517ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [520ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [521ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [533ms] refreshing cache from master, [533ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [536ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [537ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [539ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [573ms] refreshing cache from master, [573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [578ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [580ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [582ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [613ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [618ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [619ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [621ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [652ms] refreshing cache from master, [652ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [657ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [657ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [659ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [852ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [854ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [854ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [857ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [859ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [859ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [861ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1192ms] refreshing cache from master, [1193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [1193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [1196ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1196ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1198ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [1200ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1913ms] refreshing cache from master, [1913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [1914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [1914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [1916ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1916ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1918ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [1918ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [3612ms] refreshing cache from master, [3612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [3613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [3614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [3616ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [3616ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [3617ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [3618ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [4232ms] refreshing cache from master, [4232ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [4233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [4234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [4236ms] too many traces: truncated at 100 traces, deferred=Deferred@1942606593(state=PENDING, result=null, callback=(continuation of Deferred@746028317 after retry RPC after error@773142630) -> (continuation of Deferred@202216277 after retry RPC after error@836831246) -> (continuation of Deferred@644416698 after retry RPC after error@557375001) -> (continuation of Deferred@1095722576 after retry RPC after error@2027403023) -> (continuation of Deferred@385163387 after retry RPC after error@1901788330) -> (continuation of Deferred@1260218679 after retry RPC after error@246923412) -> (continuation of Deferred@838787009 after retry RPC after error@401066744) -> (continuation of Deferred@924016098 after retry RPC after error@126912756) -> (continuation of Deferred@1079467792 after retry RPC after error@1766784488) -> (continuation of Deferred@1142899054 after retry RPC after error@1196140081) -> (continuation of Deferred@1033821719 after retry RPC after error@589415921) -> (continuation of Deferred@487318730 after retry RPC after error@124498804) -> (continuation of Deferred@1288616936 after retry RPC after error@1642979789) -> (continuation of Deferred@595747982 after retry RPC after error@2032702361) -> (continuation of Deferred@1082156719 after retry RPC after error@1324156155) -> (continuation of Deferred@2130423740 after retry RPC after error@1139953569) -> (continuation of Deferred@326766 after retry RPC after error@644414025) -> (continuation of Deferred@88946551 after retry RPC after error@1418981176) -> (continuation of Deferred@1943248456 after retry RPC after error@302553289) -> (continuation of Deferred@2115167088 after retry RPC after error@900938149) -> (continuation of Deferred@1518194821 after retry RPC after error@216384979) -> (continuation of Deferred@1703259740 after retry RPC after error@1238879576) -> (continuation of Deferred@2107295007 after retry RPC after error@1601391650) -> (continuation of Deferred@399306462 after retry RPC after error@1619414130), errback=(continuation of Deferred@746028317 after retry RPC after error@773142630) -> (continuation of Deferred@202216277 after retry RPC after error@836831246) -> (continuation of Deferred@644416698 after retry RPC after error@557375001) -> (continuation of Deferred@1095722576 after retry RPC after error@2027403023) -> (continuation of Deferred@385163387 after retry RPC after error@1901788330) -> (continuation of Deferred@1260218679 after retry RPC after error@246923412) -> (continuation of Deferred@838787009 after retry RPC after error@401066744) -> (continuation of Deferred@924016098 after retry RPC after error@126912756) -> (continuation of Deferred@1079467792 after retry RPC after error@1766784488) -> (continuation of Deferred@1142899054 after retry RPC after error@1196140081) -> (continuation of Deferred@1033821719 after retry RPC after error@589415921) -> (continuation of Deferred@487318730 after retry RPC after error@124498804) -> (continuation of Deferred@1288616936 after retry RPC after error@1642979789) -> (continuation of Deferred@595747982 after retry RPC after error@2032702361) -> (continuation of Deferred@1082156719 after retry RPC after error@1324156155) -> (continuation of Deferred@2130423740 after retry RPC after error@1139953569) -> (continuation of Deferred@326766 after retry RPC after error@644414025) -> (continuation of Deferred@88946551 after retry RPC after error@1418981176) -> (continuation of Deferred@1943248456 after retry RPC after error@302553289) -> (continuation of Deferred@2115167088 after retry RPC after error@900938149) -> (continuation of Deferred@1518194821 after retry RPC after error@216384979) -> (continuation of Deferred@1703259740 after retry RPC after error@1238879576) -> (continuation of Deferred@2107295007 after retry RPC after error@1601391650) -> (continuation of Deferred@399306462 after retry RPC after error@1619414130)))
13:40:13.693 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.101.126:34059(127.28.101.126:34059)] disconnecting while in state TERMINATED
13:40:13.695 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.101.124:36159(127.28.101.124:36159)] disconnecting while in state READY
13:40:13.695 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.101.125:40157(127.28.101.125:40157)] disconnecting while in state TERMINATED
13:40:13.700 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:40:13.702 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.101.124:36159(127.28.101.124:36159)] handling channelInactive
13:40:13.702 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.101.124:36159(127.28.101.124:36159)] cleaning up while in state READY due to: connection closed
13:40:13.704 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:40:13.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:13.710183 29077 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 29436
13:40:14.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:40:14Z chronyd exiting
13:40:15.753 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:40:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:15.754361 28352 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 28633
13:40:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:15.774755 28352 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 28789
13:40:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:15.796145 28352 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 28933
13:40:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:15.818167 28352 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 28364
13:40:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:15.842617 28352 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 28434
13:40:15.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240426 13:40:15.868124 28352 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskPWbCo1/build/tsan/bin/kudu with pid 28509
13:40:16.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-04-26T13:40:16Z chronyd exiting
13:40:17.893 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29107), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [153ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [156ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [465ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [468ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [473ms] refreshing cache from master, [474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [479ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [481ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [484ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [493ms] refreshing cache from master, [493ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [494ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [494ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [496ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [498ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [499ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [499ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [513ms] refreshing cache from master, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [517ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [517ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [520ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [521ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [533ms] refreshing cache from master, [533ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [536ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [537ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [539ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [573ms] refreshing cache from master, [573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [578ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [580ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [582ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [613ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [618ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [619ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [621ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [652ms] refreshing cache from master, [652ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [657ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [657ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [659ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [852ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [854ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [854ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [857ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [859ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [859ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [861ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1192ms] refreshing cache from master, [1193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [1193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [1196ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1196ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1198ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [1200ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1913ms] refreshing cache from master, [1913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [1914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [1914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [1916ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1916ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1918ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [1918ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [3612ms] refreshing cache from master, [3612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [3613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [3614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [3616ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [3616ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [3617ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [3618ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [4232ms] refreshing cache from master, [4232ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [4233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [4234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [4236ms] too many traces: truncated at 100 traces, deferred=Deferred@1942606593(state=PENDING, result=null, callback=(continuation of Deferred@746028317 after retry RPC after error@773142630) -> (continuation of Deferred@202216277 after retry RPC after error@836831246) -> (continuation of Deferred@644416698 after retry RPC after error@557375001) -> (continuation of Deferred@1095722576 after retry RPC after error@2027403023) -> (continuation of Deferred@385163387 after retry RPC after error@1901788330) -> (continuation of Deferred@1260218679 after retry RPC after error@246923412) -> (continuation of Deferred@838787009 after retry RPC after error@401066744) -> (continuation of Deferred@924016098 after retry RPC after error@126912756) -> (continuation of Deferred@1079467792 after retry RPC after error@1766784488) -> (continuation of Deferred@1142899054 after retry RPC after error@1196140081) -> (continuation of Deferred@1033821719 after retry RPC after error@589415921) -> (continuation of Deferred@487318730 after retry RPC after error@124498804) -> (continuation of Deferred@1288616936 after retry RPC after error@1642979789) -> (continuation of Deferred@595747982 after retry RPC after error@2032702361) -> (continuation of Deferred@1082156719 after retry RPC after error@1324156155) -> (continuation of Deferred@2130423740 after retry RPC after error@1139953569) -> (continuation of Deferred@326766 after retry RPC after error@644414025) -> (continuation of Deferred@88946551 after retry RPC after error@1418981176) -> (continuation of Deferred@1943248456 after retry RPC after error@302553289) -> (continuation of Deferred@2115167088 after retry RPC after error@900938149) -> (continuation of Deferred@1518194821 after retry RPC after error@216384979) -> (continuation of Deferred@1703259740 after retry RPC after error@1238879576) -> (continuation of Deferred@2107295007 after retry RPC after error@1601391650) -> (continuation of Deferred@399306462 after retry RPC after error@1619414130), errback=(continuation of Deferred@746028317 after retry RPC after error@773142630) -> (continuation of Deferred@202216277 after retry RPC after error@836831246) -> (continuation of Deferred@644416698 after retry RPC after error@557375001) -> (continuation of Deferred@1095722576 after retry RPC after error@2027403023) -> (continuation of Deferred@385163387 after retry RPC after error@1901788330) -> (continuation of Deferred@1260218679 after retry RPC after error@246923412) -> (continuation of Deferred@838787009 after retry RPC after error@401066744) -> (continuation of Deferred@924016098 after retry RPC after error@126912756) -> (continuation of Deferred@1079467792 after retry RPC after error@1766784488) -> (continuation of Deferred@1142899054 after retry RPC after error@1196140081) -> (continuation of Deferred@1033821719 after retry RPC after error@589415921) -> (continuation of Deferred@487318730 after retry RPC after error@124498804) -> (continuation of Deferred@1288616936 after retry RPC after error@1642979789) -> (continuation of Deferred@595747982 after retry RPC after error@2032702361) -> (continuation of Deferred@1082156719 after retry RPC after error@1324156155) -> (continuation of Deferred@2130423740 after retry RPC after error@1139953569) -> (continuation of Deferred@326766 after retry RPC after error@644414025) -> (continuation of Deferred@88946551 after retry RPC after error@1418981176) -> (continuation of Deferred@1943248456 after retry RPC after error@302553289) -> (continuation of Deferred@2115167088 after retry RPC after error@900938149) -> (continuation of Deferred@1518194821 after retry RPC after error@216384979) -> (continuation of Deferred@1703259740 after retry RPC after error@1238879576) -> (continuation of Deferred@2107295007 after retry RPC after error@1601391650) -> (continuation of Deferred@399306462 after retry RPC after error@1619414130)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.101.125:40157
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29107), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [153ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [156ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [465ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [468ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [473ms] refreshing cache from master, [474ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [479ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [481ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [484ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [493ms] refreshing cache from master, [493ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [494ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [494ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [496ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [498ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [499ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [499ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [513ms] refreshing cache from master, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [517ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [517ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [520ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [521ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [533ms] refreshing cache from master, [533ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [536ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [537ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [539ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [539ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [573ms] refreshing cache from master, [573ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [575ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [578ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [580ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [582ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [613ms] refreshing cache from master, [613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [618ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [619ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [621ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [652ms] refreshing cache from master, [652ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [657ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [657ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [659ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [852ms] refreshing cache from master, [853ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [854ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [854ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [857ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [859ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [859ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [861ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1192ms] refreshing cache from master, [1193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [1193ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [1194ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [1196ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1196ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1198ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [1200ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1913ms] refreshing cache from master, [1913ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [1914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [1914ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [1916ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [1916ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [1918ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [1918ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [3612ms] refreshing cache from master, [3612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [3613ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [3614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [3616ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.125:40157: Network error: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157, [3616ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.126:34059: Network error: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [3617ms] Sub RPC ConnectToMaster: received response from server master-127.28.101.124:36159: OK, [3618ms] delaying RPC due to: Service unavailable: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059, [4232ms] refreshing cache from master, [4232ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.125:40157, [4233ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.124:36159, [4234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.101.126:34059, [4236ms] too many traces: truncated at 100 traces, deferred=Deferred@1942606593(state=PENDING, result=null, callback=(continuation of Deferred@746028317 after retry RPC after error@773142630) -> (continuation of Deferred@202216277 after retry RPC after error@836831246) -> (continuation of Deferred@644416698 after retry RPC after error@557375001) -> (continuation of Deferred@1095722576 after retry RPC after error@2027403023) -> (continuation of Deferred@385163387 after retry RPC after error@1901788330) -> (continuation of Deferred@1260218679 after retry RPC after error@246923412) -> (continuation of Deferred@838787009 after retry RPC after error@401066744) -> (continuation of Deferred@924016098 after retry RPC after error@126912756) -> (continuation of Deferred@1079467792 after retry RPC after error@1766784488) -> (continuation of Deferred@1142899054 after retry RPC after error@1196140081) -> (continuation of Deferred@1033821719 after retry RPC after error@589415921) -> (continuation of Deferred@487318730 after retry RPC after error@124498804) -> (continuation of Deferred@1288616936 after retry RPC after error@1642979789) -> (continuation of Deferred@595747982 after retry RPC after error@2032702361) -> (continuation of Deferred@1082156719 after retry RPC after error@1324156155) -> (continuation of Deferred@2130423740 after retry RPC after error@1139953569) -> (continuation of Deferred@326766 after retry RPC after error@644414025) -> (continuation of Deferred@88946551 after retry RPC after error@1418981176) -> (continuation of Deferred@1943248456 after retry RPC after error@302553289) -> (continuation of Deferred@2115167088 after retry RPC after error@900938149) -> (continuation of Deferred@1518194821 after retry RPC after error@216384979) -> (continuation of Deferred@1703259740 after retry RPC after error@1238879576) -> (continuation of Deferred@2107295007 after retry RPC after error@1601391650) -> (continuation of Deferred@399306462 after retry RPC after error@1619414130), errback=(continuation of Deferred@746028317 after retry RPC after error@773142630) -> (continuation of Deferred@202216277 after retry RPC after error@836831246) -> (continuation of Deferred@644416698 after retry RPC after error@557375001) -> (continuation of Deferred@1095722576 after retry RPC after error@2027403023) -> (continuation of Deferred@385163387 after retry RPC after error@1901788330) -> (continuation of Deferred@1260218679 after retry RPC after error@246923412) -> (continuation of Deferred@838787009 after retry RPC after error@401066744) -> (continuation of Deferred@924016098 after retry RPC after error@126912756) -> (continuation of Deferred@1079467792 after retry RPC after error@1766784488) -> (continuation of Deferred@1142899054 after retry RPC after error@1196140081) -> (continuation of Deferred@1033821719 after retry RPC after error@589415921) -> (continuation of Deferred@487318730 after retry RPC after error@124498804) -> (continuation of Deferred@1288616936 after retry RPC after error@1642979789) -> (continuation of Deferred@595747982 after retry RPC after error@2032702361) -> (continuation of Deferred@1082156719 after retry RPC after error@1324156155) -> (continuation of Deferred@2130423740 after retry RPC after error@1139953569) -> (continuation of Deferred@326766 after retry RPC after error@644414025) -> (continuation of Deferred@88946551 after retry RPC after error@1418981176) -> (continuation of Deferred@1943248456 after retry RPC after error@302553289) -> (continuation of Deferred@2115167088 after retry RPC after error@900938149) -> (continuation of Deferred@1518194821 after retry RPC after error@216384979) -> (continuation of Deferred@1703259740 after retry RPC after error@1238879576) -> (continuation of Deferred@2107295007 after retry RPC after error@1601391650) -> (continuation of Deferred@399306462 after retry RPC after error@1619414130)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.101.125:40157,127.28.101.124:36159,127.28.101.126:34059) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.126:34059(127.28.101.126:34059): Connection refused: /127.28.101.126:34059,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.101.125:40157(127.28.101.125:40157): Connection refused: /127.28.101.125:40157
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.101.125:40157
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)