23:51:21.541 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
23:51:21.553 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskvJNS2u/test-tmp
23:51:21.555 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:51:21.559 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskvJNS2u/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:51:21.759 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
23:51:21.987 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:51:21.988 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:51:21.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:21.993158 19175 mini_oidc.cc:152] Starting JWKS server
23:51:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:21.997611 19175 webserver.cc:460] Webserver started at http://127.0.0.1:39593/ using document root <none> and password file <none>
23:51:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:21.998198 19175 mini_oidc.cc:189] Starting OIDC Discovery server
23:51:22.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:22.000322 19175 webserver.cc:460] Webserver started at http://127.0.0.1:35393/ using document root <none> and password file <none>
23:51:22.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T23:51:22Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:51:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T23:51:22Z Disabled control of system clock
23:51:22.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:22.066061 19175 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/wal
23:51:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data
23:51:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/logs
23:51:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data/info.pb
23:51:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.254:44343
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.254
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39593/jwks.json with env {}
23:51:22.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.370716 19187 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:22.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.371275 19187 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:22.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.371855 19187 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.403367 19187 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.403687 19187 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.403926 19187 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.404171 19187 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.404465 19187 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.404641 19187 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39593/jwks.json
23:51:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:22.446061 19187 master_runner.cc:386] Master server non-default flags:
23:51:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data
23:51:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/wal
23:51:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.254:44343
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39593/jwks.json
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:22.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data/info.pb
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.254
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/logs
23:51:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:22.447310 19187 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:22.448997 19187 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.465626 19196 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.466145 19194 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:22.467092 19193 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:22.468266 19187 server_base.cc:1034] running on GCE node
23:51:23.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.580794 19187 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:23.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.583597 19187 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:23.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.584981 19187 hybrid_clock.cc:648] HybridClock initialized: now 1714780283584947 us; error 66 us; skew 500 ppm
23:51:23.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.585790 19187 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.592581 19187 webserver.cc:460] Webserver started at http://127.18.185.254:36149/ using document root <none> and password file <none>
23:51:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.593528 19187 fs_manager.cc:359] Metadata directory not provided
23:51:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.593725 19187 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:23.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.594100 19187 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:23.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.599335 19187 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data/instance:
23:51:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "286511041b8647658d9fc9fc3d083fea"
23:51:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:23 on dist-test-slave-4816"
23:51:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.600565 19187 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/wal/instance:
23:51:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "286511041b8647658d9fc9fc3d083fea"
23:51:23.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:23 on dist-test-slave-4816"
23:51:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.607908 19187 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
23:51:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.614157 19203 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.615279 19187 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
23:51:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.615594 19187 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/wal
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "286511041b8647658d9fc9fc3d083fea"
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:23 on dist-test-slave-4816"
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.615962 19187 fs_report.cc:383] FS layout report
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/wal
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/wal
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data/data
23:51:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:23.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.668466 19187 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.669922 19187 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.670322 19187 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:23.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.738438 19187 rpc_server.cc:307] RPC server started. Bound to: 127.18.185.254:44343
23:51:23.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.738520 19254 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.185.254:44343 every 8 connection(s)
23:51:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.741012 19187 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/data/info.pb
23:51:23.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.743000 19175 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 19187
23:51:23.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.743585 19175 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-0/wal/instance
23:51:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.746651 19255 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.747668 19175 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/wal
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/logs
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data/info.pb
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:23.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:23.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.253:37555
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.253
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39593/jwks.json with env {}
23:51:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.759521 19255 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } has no permanent_uuid. Determining permanent_uuid...
23:51:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.787505 19255 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } has no permanent_uuid. Determining permanent_uuid...
23:51:23.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:23.790019 19207 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:23.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:23.794435 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.253:37555: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.843994 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } attempt: 1
23:51:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:23.847882 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.253:37555: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:23.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:23.916291 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } attempt: 2
23:51:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:23.920213 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.253:37555: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:24.011627 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } attempt: 3
23:51:24.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.015444 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.253:37555: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:24.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.060654 19257 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:24.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.061256 19257 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:24.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.061866 19257 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.092952 19257 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.093235 19257 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.093434 19257 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:24.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.093614 19257 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.093796 19257 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.093931 19257 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39593/jwks.json
23:51:24.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:24.129175 19257 master_runner.cc:386] Master server non-default flags:
23:51:24.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:24.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/wal
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.253:37555
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39593/jwks.json
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data/info.pb
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.253
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/logs
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:24.130352 19257 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:24.132369 19257 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.148056 19267 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:24.158809 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } attempt: 4
23:51:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.162664 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.253:37555: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:24.462082 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } attempt: 5
23:51:24.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.467836 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.253:37555: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.015338 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } attempt: 6
23:51:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.022115 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.253:37555: Network error: Client connection negotiation failed: client connection to 127.18.185.253:37555: connect: Connection refused (error 111)
23:51:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.148854 19266 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:24.151563 19257 server_base.cc:1034] running on GCE node
23:51:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:24.149911 19269 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:25.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.255041 19257 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.257854 19257 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.259219 19257 hybrid_clock.cc:648] HybridClock initialized: now 1714780285259170 us; error 59 us; skew 500 ppm
23:51:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.260195 19257 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:25.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.266556 19257 webserver.cc:460] Webserver started at http://127.18.185.253:39435/ using document root <none> and password file <none>
23:51:25.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.267580 19257 fs_manager.cc:359] Metadata directory not provided
23:51:25.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.267800 19257 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:25.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.268327 19257 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.273751 19257 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data/instance:
23:51:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bee56469a4b84b669a9e0af683d62105"
23:51:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:25 on dist-test-slave-4816"
23:51:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.274984 19257 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/wal/instance:
23:51:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bee56469a4b84b669a9e0af683d62105"
23:51:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:25 on dist-test-slave-4816"
23:51:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.282235 19257 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
23:51:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.287786 19278 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.288853 19257 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
23:51:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.289247 19257 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/wal
23:51:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bee56469a4b84b669a9e0af683d62105"
23:51:25.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:25 on dist-test-slave-4816"
23:51:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.289609 19257 fs_report.cc:383] FS layout report
23:51:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/wal
23:51:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/wal
23:51:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data/data
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.343366 19257 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.344931 19257 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.345427 19257 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.417373 19257 rpc_server.cc:307] RPC server started. Bound to: 127.18.185.253:37555
23:51:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.417449 19329 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.185.253:37555 every 8 connection(s)
23:51:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.420127 19257 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/data/info.pb
23:51:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.423662 19175 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 19257
23:51:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.424170 19175 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-1/wal/instance
23:51:25.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.426223 19330 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.427639 19175 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/wal
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/logs
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data/info.pb
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.252:44491
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.252
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39593/jwks.json with env {}
23:51:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.443132 19330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } has no permanent_uuid. Determining permanent_uuid...
23:51:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.466254 19330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } has no permanent_uuid. Determining permanent_uuid...
23:51:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.480942 19330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } has no permanent_uuid. Determining permanent_uuid...
23:51:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.483700 19281 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.488050 19330 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.537580 19330 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 1
23:51:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.541764 19330 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.610175 19330 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 2
23:51:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.614041 19330 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:25.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.705402 19330 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 3
23:51:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.709188 19330 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:25.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.737509 19332 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:25.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.738054 19332 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:25.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.738503 19332 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:25.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.769420 19332 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.769763 19332 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.769985 19332 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.770227 19332 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.770455 19332 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.770617 19332 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39593/jwks.json
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.804816 19332 master_runner.cc:386] Master server non-default flags:
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/wal
23:51:25.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.252:44491
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39593/jwks.json
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data/info.pb
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.252
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/logs
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.806073 19332 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.808174 19332 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:25.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.827013 19345 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:25.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.852581 19330 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 4
23:51:25.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.858044 19330 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.082808 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } attempt: 7
23:51:26.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.098088 19255 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } has no permanent_uuid. Determining permanent_uuid...
23:51:26.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.105518 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.157449 19330 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 5
23:51:26.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.162146 19330 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.163944 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 1
23:51:26.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.169442 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.250909 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 2
23:51:26.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.255717 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.341113 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 3
23:51:26.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.345824 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.486194 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 4
23:51:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.490788 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.709612 19330 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 6
23:51:26.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.717650 19330 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.774183 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 5
23:51:26.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:26.777969 19255 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.18.185.252:44491: Network error: Client connection negotiation failed: client connection to 127.18.185.252:44491: connect: Connection refused (error 111)
23:51:26.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.827708 19343 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:26.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:25.828279 19332 server_base.cc:1034] running on GCE node
23:51:26.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:25.829120 19342 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.959877 19332 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.963101 19332 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:26.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.964521 19332 hybrid_clock.cc:648] HybridClock initialized: now 1714780286964506 us; error 49 us; skew 500 ppm
23:51:26.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.965365 19332 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:26.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.975641 19332 webserver.cc:460] Webserver started at http://127.18.185.252:38937/ using document root <none> and password file <none>
23:51:26.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.976516 19332 fs_manager.cc:359] Metadata directory not provided
23:51:26.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.976704 19332 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:26.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.977152 19332 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:26.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.984124 19332 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data/instance:
23:51:26.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ac8252148ea434d955474d411074efe"
23:51:26.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:26 on dist-test-slave-4816"
23:51:26.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.985613 19332 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/wal/instance:
23:51:26.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ac8252148ea434d955474d411074efe"
23:51:26.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:26 on dist-test-slave-4816"
23:51:26.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:26.994756 19332 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
23:51:27.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.000070 19357 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.001230 19332 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
23:51:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.001549 19332 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/wal
23:51:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ac8252148ea434d955474d411074efe"
23:51:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:26 on dist-test-slave-4816"
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.001853 19332 fs_report.cc:383] FS layout report
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/wal
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/wal
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data/data
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.055783 19332 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.057216 19332 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.057627 19332 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.127686 19332 rpc_server.cc:307] RPC server started. Bound to: 127.18.185.252:44491
23:51:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.127795 19408 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.185.252:44491 every 8 connection(s)
23:51:27.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.130275 19332 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/data/info.pb
23:51:27.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.131532 19175 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 19332
23:51:27.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.131934 19175 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/master-2/wal/instance
23:51:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.136435 19409 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:27.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.153900 19409 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } has no permanent_uuid. Determining permanent_uuid...
23:51:27.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.178563 19409 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } has no permanent_uuid. Determining permanent_uuid...
23:51:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.187438 19409 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } has no permanent_uuid. Determining permanent_uuid...
23:51:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.202632 19409 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe: Bootstrap starting.
23:51:27.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.208225 19409 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe: Neither blocks nor log segments found. Creating new log.
23:51:27.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.210359 19409 log.cc:824] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe: Log is configured to *not* fsync() on all Append() calls
23:51:27.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.214823 19409 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe: No bootstrap required, opened a new log
23:51:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.237128 19409 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.237854 19409 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.238054 19409 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5ac8252148ea434d955474d411074efe, State: Initialized, Role: FOLLOWER
23:51:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.238768 19409 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.241089 19418 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.241721 19418 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: This master's current role is: FOLLOWER
23:51:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.242408 19409 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: configured and running, proceeding with master startup.
23:51:27.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:27.267151 19429 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:27.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:27.267535 19429 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.263198 19332 master_runner.cc:186] Error getting master registration for 127.18.185.254:44343: OK, instance_id { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" instance_seqno: 1714780283718946 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:51:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.288473 19332 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:51:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:27.313148 19363 tablet.cc:2369] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:27.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.330351 19255 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 6
23:51:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.347676 19255 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: Bootstrap starting.
23:51:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.352792 19255 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: Neither blocks nor log segments found. Creating new log.
23:51:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.354490 19255 log.cc:824] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: Log is configured to *not* fsync() on all Append() calls
23:51:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.358760 19255 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: No bootstrap required, opened a new log
23:51:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.379812 19255 raft_consensus.cc:348] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.380474 19255 raft_consensus.cc:374] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.380740 19255 raft_consensus.cc:729] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 286511041b8647658d9fc9fc3d083fea, State: Initialized, Role: FOLLOWER
23:51:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.381693 19255 consensus_queue.cc:260] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.384681 19433 sys_catalog.cc:455] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.385565 19433 sys_catalog.cc:458] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [sys.catalog]: This master's current role is: FOLLOWER
23:51:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.386471 19255 sys_catalog.cc:564] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [sys.catalog]: configured and running, proceeding with master startup.
23:51:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:27.428999 19209 tablet.cc:2369] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:27.432433 19444 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:27.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:27.432785 19444 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.418671 19187 master_runner.cc:186] Error getting master registration for 127.18.185.253:37555: OK, instance_id { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" instance_seqno: 1714780285398480 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:51:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.464787 19187 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:51:27.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.653628 19418 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:27.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.654153 19418 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.656544 19418 leader_election.cc:288] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 286511041b8647658d9fc9fc3d083fea (127.18.185.254:44343), bee56469a4b84b669a9e0af683d62105 (127.18.185.253:37555)
23:51:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.658664 19230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ac8252148ea434d955474d411074efe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "286511041b8647658d9fc9fc3d083fea" is_pre_election: true
23:51:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.659595 19230 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5ac8252148ea434d955474d411074efe in term 0.
23:51:27.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.660960 19358 leader_election.cc:302] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 286511041b8647658d9fc9fc3d083fea, 5ac8252148ea434d955474d411074efe; no voters:
23:51:27.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.661979 19418 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Leader pre-election won for term 1
23:51:27.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.662304 19418 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
23:51:27.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.662600 19418 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 0 FOLLOWER]: Advancing to term 1
23:51:27.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.669489 19418 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.671613 19418 leader_election.cc:288] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [CANDIDATE]: Term 1 election: Requested vote from peers 286511041b8647658d9fc9fc3d083fea (127.18.185.254:44343), bee56469a4b84b669a9e0af683d62105 (127.18.185.253:37555)
23:51:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.674235 19230 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ac8252148ea434d955474d411074efe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "286511041b8647658d9fc9fc3d083fea"
23:51:27.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.674814 19230 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [term 0 FOLLOWER]: Advancing to term 1
23:51:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.683705 19230 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5ac8252148ea434d955474d411074efe in term 1.
23:51:27.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.687431 19358 leader_election.cc:302] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 286511041b8647658d9fc9fc3d083fea, 5ac8252148ea434d955474d411074efe; no voters:
23:51:27.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.689797 19418 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 1 FOLLOWER]: Leader election won for term 1
23:51:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.695907 19418 raft_consensus.cc:686] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [term 1 LEADER]: Becoming Leader. State: Replica: 5ac8252148ea434d955474d411074efe, State: Running, Role: LEADER
23:51:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.697474 19418 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.708729 19446 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5ac8252148ea434d955474d411074efe. Latest consensus state: current_term: 1 leader_uuid: "5ac8252148ea434d955474d411074efe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:27.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.712795 19446 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: This master's current role is: LEADER
23:51:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.719247 19448 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
23:51:27.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.737125 19448 catalog_manager.cc:1484] Initializing Kudu cluster ID...
23:51:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.777995 19330 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } attempt: 7
23:51:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.797796 19230 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [term 1 FOLLOWER]: Refusing update from remote peer 5ac8252148ea434d955474d411074efe: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:27.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.801215 19418 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [LEADER]: Connected to new peer: Peer: permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
23:51:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.823644 19330 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105: Bootstrap starting.
23:51:27.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.836145 19433 sys_catalog.cc:455] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5ac8252148ea434d955474d411074efe. Latest consensus state: current_term: 1 leader_uuid: "5ac8252148ea434d955474d411074efe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.841782 19330 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105: Neither blocks nor log segments found. Creating new log.
23:51:27.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.845762 19433 sys_catalog.cc:458] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [sys.catalog]: This master's current role is: FOLLOWER
23:51:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.847239 19330 log.cc:824] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105: Log is configured to *not* fsync() on all Append() calls
23:51:27.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.852541 19449 mvcc.cc:205] Tried to move back new op lower bound from 7023740058780823552 to 7023740058420342784. Current Snapshot: MvccSnapshot[applied={T|T < 7023740058780823552}]
23:51:27.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.871588 19330 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105: No bootstrap required, opened a new log
23:51:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.882647 19433 sys_catalog.cc:455] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5ac8252148ea434d955474d411074efe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.883370 19433 sys_catalog.cc:458] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea [sys.catalog]: This master's current role is: FOLLOWER
23:51:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.897049 19448 catalog_manager.cc:1347] Generated new cluster ID: 04a1f5dc048a4bb78f935ab7b651c597
23:51:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.897354 19448 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
23:51:27.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.900915 19446 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5ac8252148ea434d955474d411074efe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:27.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.901669 19446 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: This master's current role is: LEADER
23:51:27.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.953163 19330 raft_consensus.cc:348] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.954025 19330 raft_consensus.cc:374] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:27.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.954301 19330 raft_consensus.cc:729] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bee56469a4b84b669a9e0af683d62105, State: Initialized, Role: FOLLOWER
23:51:27.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.955157 19330 consensus_queue.cc:260] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } }
23:51:27.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.963222 19456 sys_catalog.cc:455] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:27.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.964040 19456 sys_catalog.cc:458] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [sys.catalog]: This master's current role is: FOLLOWER
23:51:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.966776 19330 sys_catalog.cc:564] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [sys.catalog]: configured and running, proceeding with master startup.
23:51:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.967833 19303 raft_consensus.cc:3037] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [term 0 FOLLOWER]: Advancing to term 1
23:51:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.968556 19305 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ac8252148ea434d955474d411074efe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bee56469a4b84b669a9e0af683d62105" is_pre_election: true
23:51:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.966961 19304 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5ac8252148ea434d955474d411074efe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bee56469a4b84b669a9e0af683d62105"
23:51:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.992180 19303 raft_consensus.cc:1261] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [term 1 FOLLOWER]: Refusing update from remote peer 5ac8252148ea434d955474d411074efe: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
23:51:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.995139 19448 catalog_manager.cc:1370] Generated new certificate authority record
23:51:27.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.997244 19448 catalog_manager.cc:1504] Loading token signing keys...
23:51:27.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:27.997123 19418 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [LEADER]: Connected to new peer: Peer: permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
23:51:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.036697 19448 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe: Generated new TSK 0
23:51:28.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.037899 19448 catalog_manager.cc:1514] Initializing in-progress tserver states...
23:51:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.067459 19446 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5ac8252148ea434d955474d411074efe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.068234 19446 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5ac8252148ea434d955474d411074efe [sys.catalog]: This master's current role is: LEADER
23:51:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.070580 19456 sys_catalog.cc:455] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5ac8252148ea434d955474d411074efe. Latest consensus state: current_term: 1 leader_uuid: "5ac8252148ea434d955474d411074efe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:28.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.071301 19456 sys_catalog.cc:458] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [sys.catalog]: This master's current role is: FOLLOWER
23:51:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.086670 19464 sys_catalog.cc:455] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5ac8252148ea434d955474d411074efe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "286511041b8647658d9fc9fc3d083fea" member_type: VOTER last_known_addr { host: "127.18.185.254" port: 44343 } } peers { permanent_uuid: "bee56469a4b84b669a9e0af683d62105" member_type: VOTER last_known_addr { host: "127.18.185.253" port: 37555 } } peers { permanent_uuid: "5ac8252148ea434d955474d411074efe" member_type: VOTER last_known_addr { host: "127.18.185.252" port: 44491 } } }
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.087756 19175 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/wal
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/logs
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data/info.pb
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.193:0
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.185.193
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.193
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.087795 19464 sys_catalog.cc:458] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105 [sys.catalog]: This master's current role is: FOLLOWER
23:51:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.121165 19473 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
23:51:28.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.438500 19444 catalog_manager.cc:1259] Loaded cluster ID: 04a1f5dc048a4bb78f935ab7b651c597
23:51:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.438805 19444 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: loading cluster ID for follower catalog manager: success
23:51:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.446465 19444 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: acquiring CA information for follower catalog manager: success
23:51:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.451637 19444 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 286511041b8647658d9fc9fc3d083fea: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:51:28.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:28.572301 19471 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:28.572957 19471 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:28.573740 19471 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:28.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:28.629334 19471 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:28.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:28.630690 19471 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.185.193
23:51:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.690941 19471 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/wal
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.193:0
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data/info.pb
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.193
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:28.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.185.193
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/logs
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:28.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.692428 19471 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:28.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:28.694378 19471 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:28.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:28.714962 19479 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:29.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:29.162587 19473 catalog_manager.cc:1259] Loaded cluster ID: 04a1f5dc048a4bb78f935ab7b651c597
23:51:29.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:29.167405 19473 catalog_manager.cc:1552] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105: loading cluster ID for follower catalog manager: success
23:51:29.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:29.184680 19473 catalog_manager.cc:1574] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105: acquiring CA information for follower catalog manager: success
23:51:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:29.214939 19473 catalog_manager.cc:1602] T 00000000000000000000000000000000 P bee56469a4b84b669a9e0af683d62105: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
23:51:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:30.111660 19478 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 19471
23:51:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.145323 19478 kernel_stack_watchdog.cc:198] Thread 19471 stuck at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:637 for 394ms:
23:51:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:51:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:51:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:51:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:51:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:28.716317 19480 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.146616 19471 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.431s user 0.688s sys 1.289s
23:51:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.147080 19471 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.432s user 0.688s sys 1.289s
23:51:31.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.150220 19482 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:31.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.154111 19481 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2425 milliseconds
23:51:31.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.154282 19471 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:31.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.155544 19471 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.157842 19471 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.159225 19471 hybrid_clock.cc:648] HybridClock initialized: now 1714780291159173 us; error 45 us; skew 500 ppm
23:51:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.160193 19471 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:31.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.166685 19471 webserver.cc:460] Webserver started at http://127.18.185.193:44791/ using document root <none> and password file <none>
23:51:31.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.167898 19471 fs_manager.cc:359] Metadata directory not provided
23:51:31.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.168154 19471 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:31.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.168716 19471 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.173585 19471 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data/instance:
23:51:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bc19ef6bbeab4d5cbd527c04bf181756"
23:51:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:31 on dist-test-slave-4816"
23:51:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.174738 19471 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/wal/instance:
23:51:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bc19ef6bbeab4d5cbd527c04bf181756"
23:51:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:31 on dist-test-slave-4816"
23:51:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.182408 19471 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
23:51:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.189606 19492 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.192951 19471 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.000s
23:51:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.193339 19471 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/wal
23:51:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bc19ef6bbeab4d5cbd527c04bf181756"
23:51:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:31 on dist-test-slave-4816"
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.193704 19471 fs_report.cc:383] FS layout report
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/wal
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/wal
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data/data
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.254701 19471 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.256175 19471 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.256651 19471 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:31.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.259639 19471 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:51:31.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.264369 19471 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:31.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.264564 19471 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:51:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.264896 19471 ts_tablet_manager.cc:590] Registered 0 tablets
23:51:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.265081 19471 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
23:51:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.451654 19471 rpc_server.cc:307] RPC server started. Bound to: 127.18.185.193:38763
23:51:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.451790 19604 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.185.193:38763 every 8 connection(s)
23:51:31.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.455112 19471 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/data/info.pb
23:51:31.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.464540 19175 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 19471
23:51:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.465322 19175 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-0/wal/instance
23:51:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.477440 19175 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/wal
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/logs
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data/info.pb
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.194:0
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.185.194
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.194
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:31.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.531359 19611 heartbeater.cc:347] Connected to a master server at 127.18.185.254:44343
23:51:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.531827 19611 heartbeater.cc:464] Registering TS with master...
23:51:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.532557 19605 heartbeater.cc:347] Connected to a master server at 127.18.185.253:37555
23:51:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.532932 19605 heartbeater.cc:464] Registering TS with master...
23:51:31.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.533146 19611 heartbeater.cc:511] Master 127.18.185.254:44343 requested a full tablet report, sending...
23:51:31.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.533975 19605 heartbeater.cc:511] Master 127.18.185.253:37555 requested a full tablet report, sending...
23:51:31.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.536861 19220 ts_manager.cc:194] Registered new tserver with Master: bc19ef6bbeab4d5cbd527c04bf181756 (127.18.185.193:38763)
23:51:31.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.537274 19295 ts_manager.cc:194] Registered new tserver with Master: bc19ef6bbeab4d5cbd527c04bf181756 (127.18.185.193:38763)
23:51:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.540113 19606 heartbeater.cc:347] Connected to a master server at 127.18.185.252:44491
23:51:31.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.540406 19606 heartbeater.cc:464] Registering TS with master...
23:51:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.541182 19606 heartbeater.cc:511] Master 127.18.185.252:44491 requested a full tablet report, sending...
23:51:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.543224 19374 ts_manager.cc:194] Registered new tserver with Master: bc19ef6bbeab4d5cbd527c04bf181756 (127.18.185.193:38763)
23:51:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.545302 19374 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.185.193:37641
23:51:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.835289 19615 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.835762 19615 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:31.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.836253 19615 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.867851 19615 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.868635 19615 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.185.194
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.903788 19615 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/wal
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.194:0
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data/info.pb
23:51:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.194
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.185.194
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/logs
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.905303 19615 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.907505 19615 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.930604 19623 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:32.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:32.548565 19606 heartbeater.cc:503] Master 127.18.185.252:44491 was elected leader, sending a full tablet report...
23:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.934423 19622 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:31.935508 19625 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:33.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:31.936584 19615 server_base.cc:1034] running on GCE node
23:51:33.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.059265 19615 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:33.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.062085 19615 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:33.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.063449 19615 hybrid_clock.cc:648] HybridClock initialized: now 1714780293063417 us; error 56 us; skew 500 ppm
23:51:33.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.064226 19615 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:33.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.076246 19615 webserver.cc:460] Webserver started at http://127.18.185.194:46483/ using document root <none> and password file <none>
23:51:33.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.077356 19615 fs_manager.cc:359] Metadata directory not provided
23:51:33.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.077654 19615 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.078163 19615 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:33.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.083570 19615 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data/instance:
23:51:33.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8944bb657708430f937d9fdc942fb5ed"
23:51:33.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:33 on dist-test-slave-4816"
23:51:33.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.084731 19615 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/wal/instance:
23:51:33.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8944bb657708430f937d9fdc942fb5ed"
23:51:33.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:33 on dist-test-slave-4816"
23:51:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.092785 19615 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
23:51:33.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.098615 19633 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:33.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.099656 19615 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
23:51:33.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.100224 19615 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/wal
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8944bb657708430f937d9fdc942fb5ed"
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:33 on dist-test-slave-4816"
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.100947 19615 fs_report.cc:383] FS layout report
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/wal
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/wal
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data/data
23:51:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:33.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.162222 19615 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.163800 19615 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:33.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.164331 19615 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:33.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.166882 19615 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:51:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.171140 19615 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.171360 19615 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:51:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.171666 19615 ts_tablet_manager.cc:590] Registered 0 tablets
23:51:33.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.171845 19615 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:51:33.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.304369 19615 rpc_server.cc:307] RPC server started. Bound to: 127.18.185.194:34145
23:51:33.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.304503 19745 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.185.194:34145 every 8 connection(s)
23:51:33.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.306941 19615 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/data/info.pb
23:51:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.317641 19175 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 19615
23:51:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.318432 19175 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-1/wal/instance
23:51:33.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.331285 19175 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/wal
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/logs
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data/info.pb
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.195:0
23:51:33.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.185.195
23:51:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.195
23:51:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
23:51:33.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.360463 19746 heartbeater.cc:347] Connected to a master server at 127.18.185.253:37555
23:51:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.360625 19747 heartbeater.cc:347] Connected to a master server at 127.18.185.252:44491
23:51:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.361001 19746 heartbeater.cc:464] Registering TS with master...
23:51:33.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.361092 19747 heartbeater.cc:464] Registering TS with master...
23:51:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.362419 19746 heartbeater.cc:511] Master 127.18.185.253:37555 requested a full tablet report, sending...
23:51:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.362440 19747 heartbeater.cc:511] Master 127.18.185.252:44491 requested a full tablet report, sending...
23:51:33.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.362818 19748 heartbeater.cc:347] Connected to a master server at 127.18.185.254:44343
23:51:33.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.363147 19748 heartbeater.cc:464] Registering TS with master...
23:51:33.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.364032 19748 heartbeater.cc:511] Master 127.18.185.254:44343 requested a full tablet report, sending...
23:51:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.365345 19374 ts_manager.cc:194] Registered new tserver with Master: 8944bb657708430f937d9fdc942fb5ed (127.18.185.194:34145)
23:51:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.366437 19295 ts_manager.cc:194] Registered new tserver with Master: 8944bb657708430f937d9fdc942fb5ed (127.18.185.194:34145)
23:51:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.366722 19220 ts_manager.cc:194] Registered new tserver with Master: 8944bb657708430f937d9fdc942fb5ed (127.18.185.194:34145)
23:51:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.366951 19374 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.185.194:54399
23:51:33.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.665637 19756 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:33.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.666105 19756 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:33.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.666543 19756 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.698132 19756 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.698968 19756 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.185.195
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.734119 19756 tablet_server_runner.cc:78] Tablet server non-default flags:
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.185.212:45991
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/wal
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.185.195:0
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data/info.pb
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.185.195
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.185.254:44343,127.18.185.253:37555,127.18.185.252:44491
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.185.195
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/logs
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.735298 19756 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.737087 19756 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:33.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.756707 19763 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:34.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.370364 19747 heartbeater.cc:503] Master 127.18.185.252:44491 was elected leader, sending a full tablet report...
23:51:34.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:33.759776 19756 server_base.cc:1034] running on GCE node
23:51:34.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.758316 19766 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:34.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:33.757304 19764 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:34.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.872450 19756 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.875738 19756 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.877208 19756 hybrid_clock.cc:648] HybridClock initialized: now 1714780294877169 us; error 66 us; skew 500 ppm
23:51:34.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.877980 19756 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:34.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.884023 19756 webserver.cc:460] Webserver started at http://127.18.185.195:38227/ using document root <none> and password file <none>
23:51:34.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.884958 19756 fs_manager.cc:359] Metadata directory not provided
23:51:34.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.885159 19756 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:34.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.885543 19756 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.890517 19756 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data/instance:
23:51:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79070674aa92490da6035431ccf8be6b"
23:51:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:34 on dist-test-slave-4816"
23:51:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.891551 19756 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/wal/instance:
23:51:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79070674aa92490da6035431ccf8be6b"
23:51:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:34 on dist-test-slave-4816"
23:51:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.898507 19756 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
23:51:34.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.903947 19774 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.905113 19756 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
23:51:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.905400 19756 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/wal
23:51:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "79070674aa92490da6035431ccf8be6b"
23:51:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:34 on dist-test-slave-4816"
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.905707 19756 fs_report.cc:383] FS layout report
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/wal
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/wal
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data/data
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:34.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.959568 19756 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.961072 19756 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.961570 19756 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:34.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.964161 19756 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
23:51:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.969094 19756 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
23:51:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.969285 19756 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
23:51:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.969504 19756 ts_tablet_manager.cc:590] Registered 0 tablets
23:51:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:34.969640 19756 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
23:51:35.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.101106 19756 rpc_server.cc:307] RPC server started. Bound to: 127.18.185.195:44837
23:51:35.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.101212 19886 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.185.195:44837 every 8 connection(s)
23:51:35.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.104425 19756 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/data/info.pb
23:51:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.108901 19175 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 19756
23:51:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.109529 19175 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster3063387515093643947/ts-2/wal/instance
23:51:35.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.177700 19888 heartbeater.cc:347] Connected to a master server at 127.18.185.252:44491
23:51:35.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.178237 19888 heartbeater.cc:464] Registering TS with master...
23:51:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.179776 19888 heartbeater.cc:511] Master 127.18.185.252:44491 requested a full tablet report, sending...
23:51:35.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.181371 19887 heartbeater.cc:347] Connected to a master server at 127.18.185.253:37555
23:51:35.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.181731 19887 heartbeater.cc:464] Registering TS with master...
23:51:35.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.182456 19374 ts_manager.cc:194] Registered new tserver with Master: 79070674aa92490da6035431ccf8be6b (127.18.185.195:44837)
23:51:35.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.182601 19887 heartbeater.cc:511] Master 127.18.185.253:37555 requested a full tablet report, sending...
23:51:35.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.184134 19374 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.185.195:57093
23:51:35.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.186098 19295 ts_manager.cc:194] Registered new tserver with Master: 79070674aa92490da6035431ccf8be6b (127.18.185.195:44837)
23:51:35.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.191529 19889 heartbeater.cc:347] Connected to a master server at 127.18.185.254:44343
23:51:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.191900 19889 heartbeater.cc:464] Registering TS with master...
23:51:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.192564 19889 heartbeater.cc:511] Master 127.18.185.254:44343 requested a full tablet report, sending...
23:51:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.194845 19220 ts_manager.cc:194] Registered new tserver with Master: 79070674aa92490da6035431ccf8be6b (127.18.185.195:44837)
23:51:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.207744 19175 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
23:51:35.219 [DEBUG - main] (MiniKuduCluster.java:195) Response:
23:51:35.220 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
23:51:35.230 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.185.254"
port: 44343
}
bound_http_address {
host: "127.18.185.254"
port: 36149
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.185.253"
port: 37555
}
bound_http_address {
host: "127.18.185.253"
port: 39435
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.185.252"
port: 44491
}
bound_http_address {
host: "127.18.185.252"
port: 38937
}
}
}
23:51:35.234 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:51:35.237 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.18.185.193"
port: 38763
}
bound_http_address {
host: "127.18.185.193"
port: 44791
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.18.185.194"
port: 34145
}
bound_http_address {
host: "127.18.185.194"
port: 46483
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.18.185.195"
port: 44837
}
bound_http_address {
host: "127.18.185.195"
port: 38227
}
}
}
23:51:35.238 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
23:51:35.412 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
23:51:35.529 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskvJNS2u/test-tmp
23:51:35.530 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
23:51:35.531 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskvJNS2u/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
23:51:35.534 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
23:51:35.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:35.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
23:51:35.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.880693 19898 mini_oidc.cc:152] Starting JWKS server
23:51:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.884198 19898 webserver.cc:460] Webserver started at http://127.0.0.1:36203/ using document root <none> and password file <none>
23:51:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.884613 19898 mini_oidc.cc:189] Starting OIDC Discovery server
23:51:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:35.886332 19898 webserver.cc:460] Webserver started at http://127.0.0.1:40511/ using document root <none> and password file <none>
23:51:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
23:51:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/principal' for realm 'KRBTEST.COM',
23:51:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
23:51:35.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:35 dist-test-slave-4816 krb5kdc[19911](info): setting up network...
23:51:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
23:51:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:35 dist-test-slave-4816 krb5kdc[19911](info): set up 2 sockets
23:51:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:35 dist-test-slave-4816 krb5kdc[19911](info): commencing operation
23:51:35.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
23:51:36.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:36.188798 19888 heartbeater.cc:503] Master 127.18.185.252:44491 was elected leader, sending a full tablet report...
23:51:36.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:36.955518 19898 mini_kdc.cc:121] Time spent starting KDC: real 1.028s user 0.003s sys 0.018s
23:51:36.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
23:51:36.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:51:36.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
23:51:36.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
23:51:37.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:51:37.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
23:51:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
23:51:37.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:51:37.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
23:51:37.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
23:51:37.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/test-user.keytab.
23:51:37.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/test-user.keytab.
23:51:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:37 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780297, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:51:37.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T23:51:37Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
23:51:37.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T23:51:37Z Disabled control of system clock
23:51:37.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.110.190@KRBTEST.COM; defaulting to no policy
23:51:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:51:37.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.110.190@KRBTEST.COM" created.
23:51:37.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:51:37.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.110.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.190.keytab.
23:51:37.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.110.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.190.keytab.
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:37.157817 19898 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/logs
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/info.pb
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.190:39121
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.190
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.190.keytab
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.190
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:51:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.462601 19924 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.463137 19924 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.463701 19924 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.494941 19924 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:51:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.495178 19924 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.495438 19924 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.495643 19924 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.495918 19924 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.496203 19924 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.496367 19924 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36203/jwks.json
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:37.531570 19924 master_runner.cc:386] Master server non-default flags:
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.190.keytab
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.190
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.190:39121
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/info.pb
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.190
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/logs
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:37.532796 19924 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:37.533766 19924 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:37.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.549085 19930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.549811 19931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:37.551895 19933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:37.552781 19924 server_base.cc:1034] running on GCE node
23:51:38.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.679447 19924 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.682482 19924 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:38.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.683918 19924 hybrid_clock.cc:648] HybridClock initialized: now 1714780298683875 us; error 54 us; skew 500 ppm
23:51:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.684809 19924 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:38 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780298, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.694054 19924 init.cc:377] Logged in from keytab as kudu/127.19.110.190@KRBTEST.COM (short username kudu)
23:51:38.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.699215 19924 webserver.cc:460] Webserver started at http://127.19.110.190:33469/ using document root <none> and password file <none>
23:51:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.700120 19924 fs_manager.cc:359] Metadata directory not provided
23:51:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.700345 19924 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.700820 19924 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.705724 19924 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/instance:
23:51:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd384e03671b42a3a174d4d4eb4fde86"
23:51:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:38 on dist-test-slave-4816"
23:51:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.706844 19924 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal/instance:
23:51:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd384e03671b42a3a174d4d4eb4fde86"
23:51:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:38 on dist-test-slave-4816"
23:51:38.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.713923 19924 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
23:51:38.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.719201 19942 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.720131 19924 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
23:51:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.720469 19924 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd384e03671b42a3a174d4d4eb4fde86"
23:51:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:38 on dist-test-slave-4816"
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.720803 19924 fs_report.cc:383] FS layout report
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/data
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.810712 19924 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.816134 19924 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:38.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.816680 19924 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:38.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.885480 19924 rpc_server.cc:307] RPC server started. Bound to: 127.19.110.190:39121
23:51:38.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.885555 19993 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.110.190:39121 every 8 connection(s)
23:51:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.888337 19924 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/info.pb
23:51:38.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.891016 19898 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 19924
23:51:38.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.891547 19898 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal/instance
23:51:38.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.894487 19994 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:38.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.907999 19994 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } has no permanent_uuid. Determining permanent_uuid...
23:51:38.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.110.189@KRBTEST.COM; defaulting to no policy
23:51:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:51:38.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.110.189@KRBTEST.COM" created.
23:51:38.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:38 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780298, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.190@KRBTEST.COM for kudu/127.19.110.190@KRBTEST.COM
23:51:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:51:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.110.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.189.keytab.
23:51:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.110.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.189.keytab.
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.943233 19898 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/logs
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/info.pb
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.189:44417
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.189
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.189.keytab
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.189
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:51:38.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:38.947199 19994 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } has no permanent_uuid. Determining permanent_uuid...
23:51:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:38.949819 19946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:38.954095 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.189:44417: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.003635 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } attempt: 1
23:51:39.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.007400 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.189:44417: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:39.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.075842 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } attempt: 2
23:51:39.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.079512 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.189:44417: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:39.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.170965 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } attempt: 3
23:51:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.174506 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.189:44417: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.255168 20000 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.255688 20000 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.256289 20000 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:39.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.286650 20000 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:51:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.286873 20000 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.287196 20000 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.287461 20000 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.287668 20000 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.287859 20000 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:39.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.288002 20000 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36203/jwks.json
23:51:39.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.317962 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } attempt: 4
23:51:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.321911 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.189:44417: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:39.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.331562 20000 master_runner.cc:386] Master server non-default flags:
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.189.keytab
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.189
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.189:44417
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/info.pb
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.189
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/logs
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.333015 20000 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.334329 20000 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.351477 20010 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:39.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.621276 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } attempt: 5
23:51:39.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.626066 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.189:44417: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:40.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.173404 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } attempt: 6
23:51:40.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:40.181355 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.189:44417: Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.352365 20007 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:40.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:39.353111 20008 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:40.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:39.354182 20000 server_base.cc:1034] running on GCE node
23:51:40.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.483923 20000 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:40.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.487023 20000 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:40.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.488453 20000 hybrid_clock.cc:648] HybridClock initialized: now 1714780300488407 us; error 64 us; skew 500 ppm
23:51:40.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.489327 20000 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:40.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:40 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780300, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:40.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.495028 20000 init.cc:377] Logged in from keytab as kudu/127.19.110.189@KRBTEST.COM (short username kudu)
23:51:40.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.500478 20000 webserver.cc:460] Webserver started at http://127.19.110.189:34575/ using document root <none> and password file <none>
23:51:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.501415 20000 fs_manager.cc:359] Metadata directory not provided
23:51:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.501600 20000 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.501976 20000 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.506738 20000 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/instance:
23:51:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed4760ce90f144bca111b1e65d544c6a"
23:51:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:40 on dist-test-slave-4816"
23:51:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.507761 20000 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal/instance:
23:51:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed4760ce90f144bca111b1e65d544c6a"
23:51:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:40 on dist-test-slave-4816"
23:51:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.514822 20000 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.007s
23:51:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.519963 20020 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.521019 20000 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
23:51:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.521368 20000 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed4760ce90f144bca111b1e65d544c6a"
23:51:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:40 on dist-test-slave-4816"
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.521770 20000 fs_report.cc:383] FS layout report
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/data
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:40.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.578980 20000 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:40.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.584558 20000 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:40.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.585014 20000 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.661780 20000 rpc_server.cc:307] RPC server started. Bound to: 127.19.110.189:44417
23:51:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.661854 20071 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.110.189:44417 every 8 connection(s)
23:51:40.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.664465 20000 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/info.pb
23:51:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.669857 20072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.674950 19898 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 20000
23:51:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.675323 19898 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal/instance
23:51:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.680843 20072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } has no permanent_uuid. Determining permanent_uuid...
23:51:40.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.110.188@KRBTEST.COM; defaulting to no policy
23:51:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:51:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.110.188@KRBTEST.COM" created.
23:51:40.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:40 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780300, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.189@KRBTEST.COM for kudu/127.19.110.190@KRBTEST.COM
23:51:40.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.716058 20072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } has no permanent_uuid. Determining permanent_uuid...
23:51:40.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
23:51:40.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.110.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.188.keytab.
23:51:40.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.110.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.188.keytab.
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.721915 19898 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/logs
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/info.pb
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:40.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.188:38311
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.188
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.188.keytab
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.188
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:40.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:51:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:40 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780300, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.189@KRBTEST.COM for kudu/127.19.110.189@KRBTEST.COM
23:51:40.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.743387 20072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } has no permanent_uuid. Determining permanent_uuid...
23:51:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:40.745945 20021 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:40.750677 20072 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:40.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.800170 20072 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 1
23:51:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:40.803735 20072 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:40.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.872262 20072 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 2
23:51:40.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:40.875990 20072 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:40.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:40.967458 20072 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 3
23:51:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:40.971119 20072 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.029163 20079 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.029721 20079 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:41.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.030313 20079 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.061914 20079 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:51:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.062253 20079 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.062518 20079 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:41.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.062731 20079 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:41.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.062923 20079 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:41.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.063112 20079 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:41.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.063264 20079 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36203/jwks.json
23:51:41.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.099476 20079 master_runner.cc:386] Master server non-default flags:
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.188.keytab
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.188
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.188:38311
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:41.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/info.pb
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.188
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/logs
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.100632 20079 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:41.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.101624 20079 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.114544 20072 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 4
23:51:41.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.118292 20086 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:41.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.121325 20072 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.241755 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } attempt: 7
23:51:41.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:41 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780298, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.190@KRBTEST.COM for kudu/127.19.110.189@KRBTEST.COM
23:51:41.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.295836 19994 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } has no permanent_uuid. Determining permanent_uuid...
23:51:41.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.304955 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.363569 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 1
23:51:41.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.370527 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.420847 20072 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 5
23:51:41.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.428164 20072 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.452163 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 2
23:51:41.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.487979 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.574599 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 3
23:51:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.579176 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.719681 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 4
23:51:41.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.726656 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:41.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:41.975732 20072 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 6
23:51:41.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.987140 20072 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:42.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.010334 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 5
23:51:42.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.016789 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.379159 20088 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:51:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:41.119120 20087 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:42.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.383435 20089 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:42.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.384593 20079 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.268s user 0.482s sys 0.774s
23:51:42.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.384974 20079 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.268s user 0.486s sys 0.774s
23:51:42.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.385263 20079 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:42.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.387897 20079 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:42.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.390622 20079 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:42.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.392001 20079 hybrid_clock.cc:648] HybridClock initialized: now 1714780302391949 us; error 58 us; skew 500 ppm
23:51:42.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.392990 20079 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:42.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780302, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.400638 20079 init.cc:377] Logged in from keytab as kudu/127.19.110.188@KRBTEST.COM (short username kudu)
23:51:42.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.408401 20079 webserver.cc:460] Webserver started at http://127.19.110.188:35901/ using document root <none> and password file <none>
23:51:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.409710 20079 fs_manager.cc:359] Metadata directory not provided
23:51:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.410002 20079 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:42.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.410558 20079 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
23:51:42.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.418602 20079 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/instance:
23:51:42.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bd30c11cf524c1791f300c60a5485e4"
23:51:42.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:42 on dist-test-slave-4816"
23:51:42.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.420382 20079 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal/instance:
23:51:42.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bd30c11cf524c1791f300c60a5485e4"
23:51:42.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:42 on dist-test-slave-4816"
23:51:42.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.430899 20079 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
23:51:42.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.439229 20106 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:42.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.440481 20079 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.440778 20079 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bd30c11cf524c1791f300c60a5485e4"
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:42 on dist-test-slave-4816"
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.441175 20079 fs_report.cc:383] FS layout report
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/data
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.545290 20079 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.550544 20079 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:42.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.550999 20079 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.569417 19994 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } attempt: 6
23:51:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.578138 19994 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.19.110.188:38311: Network error: Client connection negotiation failed: client connection to 127.19.110.188:38311: connect: Connection refused (error 111)
23:51:42.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.626745 20079 rpc_server.cc:307] RPC server started. Bound to: 127.19.110.188:38311
23:51:42.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.626844 20159 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.110.188:38311 every 8 connection(s)
23:51:42.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.629477 20079 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/info.pb
23:51:42.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.634917 20160 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
23:51:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.636296 19898 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 20079
23:51:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.636689 19898 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal/instance
23:51:42.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.649204 20160 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } has no permanent_uuid. Determining permanent_uuid...
23:51:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780302, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for kudu/127.19.110.190@KRBTEST.COM
23:51:42.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780297, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.110.188@KRBTEST.COM
23:51:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.684207 20160 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } has no permanent_uuid. Determining permanent_uuid...
23:51:42.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780302, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for kudu/127.19.110.189@KRBTEST.COM
23:51:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780297, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.110.189@KRBTEST.COM
23:51:42.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.706964 20160 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } has no permanent_uuid. Determining permanent_uuid...
23:51:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780302, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for kudu/127.19.110.188@KRBTEST.COM
23:51:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780297, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.110.190@KRBTEST.COM
23:51:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.734647 20160 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Bootstrap starting.
23:51:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.738720 19898 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
23:51:42.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.740732 20160 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Neither blocks nor log segments found. Creating new log.
23:51:42.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.742619 20160 log.cc:824] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Log is configured to *not* fsync() on all Append() calls
23:51:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.747334 20160 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: No bootstrap required, opened a new log
23:51:42.749 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:42.750 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
23:51:42.765 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
23:51:42.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
23:51:42.770 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.110.190"
port: 39121
}
bound_http_address {
host: "127.19.110.190"
port: 33469
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.110.189"
port: 44417
}
bound_http_address {
host: "127.19.110.189"
port: 34575
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.110.188"
port: 38311
}
bound_http_address {
host: "127.19.110.188"
port: 35901
}
}
}
23:51:42.771 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
23:51:42.771 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:42.771 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:51:42.773 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:51:42.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.778921 20160 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:42.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.779881 20160 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:42.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.780208 20160 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3bd30c11cf524c1791f300c60a5485e4, State: Initialized, Role: FOLLOWER
23:51:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.781380 20160 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:42.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.784584 20170 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } } }
23:51:42.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.785459 20170 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [sys.catalog]: This master's current role is: FOLLOWER
23:51:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.787109 20160 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [sys.catalog]: configured and running, proceeding with master startup.
23:51:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.806021 20112 tablet.cc:2369] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780302, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:51:42.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.827370 20182 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:42.827653 20182 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:42.828 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc
23:51:42.835 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:51:42.836 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc
23:51:42.838 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:51:42.841 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
23:51:42.843 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
23:51:42.844 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
23:51:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.814122 20079 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: OK, instance_id { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" instance_seqno: 1714780298867031 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
23:51:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.850124 20079 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:51:42.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:42 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780302, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:42.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
23:51:42.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:42.876 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc
23:51:42.877 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
23:51:42.878 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.110.188:38311
23:51:42.879 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
23:51:42.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.880088 19898 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 20079
23:51:42.907 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:42.908 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.110.190:39121
23:51:42.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
23:51:42.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.909425 19898 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 19924
23:51:42.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:42.929 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.110.189:44417
23:51:42.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
23:51:42.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.929641 19898 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 20000
23:51:42.950 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:42.950 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.110.188:38311
23:51:42.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:42.953713 19898 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/logs
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/info.pb
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:42.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.188:38311
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.188
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35901
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.188.keytab
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.188
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:51:43.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.263084 20186 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:43.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.263634 20186 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.264247 20186 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:43.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.294675 20186 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:51:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.294914 20186 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.295207 20186 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.295409 20186 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.295600 20186 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.295785 20186 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:43.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.295923 20186 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36203/jwks.json
23:51:43.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:43.331847 20186 master_runner.cc:386] Master server non-default flags:
23:51:43.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:43.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.188.keytab
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.188
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.188:38311
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/info.pb
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.188
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35901
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/logs
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:43.332980 20186 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:43.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:43.333896 20186 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:43.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.350997 20192 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.351657 20193 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:43.352344 20195 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:43.353642 20186 server_base.cc:1034] running on GCE node
23:51:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.507586 20186 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:44.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.510156 20186 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.511588 20186 hybrid_clock.cc:648] HybridClock initialized: now 1714780304511513 us; error 82 us; skew 500 ppm
23:51:44.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.512357 20186 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:44.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:44 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780304, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:44.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.522547 20186 init.cc:377] Logged in from keytab as kudu/127.19.110.188@KRBTEST.COM (short username kudu)
23:51:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.527833 20186 webserver.cc:460] Webserver started at http://127.19.110.188:35901/ using document root <none> and password file <none>
23:51:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.528772 20186 fs_manager.cc:359] Metadata directory not provided
23:51:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.529090 20186 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.536993 20186 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.005s
23:51:44.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.541775 20205 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.542905 20186 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
23:51:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.543226 20186 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bd30c11cf524c1791f300c60a5485e4"
23:51:44.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:42 on dist-test-slave-4816"
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.545655 20186 fs_report.cc:383] FS layout report
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/wal
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/data
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:44.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.605094 20186 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.610435 20186 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.610854 20186 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.679672 20186 rpc_server.cc:307] RPC server started. Bound to: 127.19.110.188:38311
23:51:44.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.679865 20256 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.110.188:38311 every 8 connection(s)
23:51:44.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.682511 20186 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-2/data/info.pb
23:51:44.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.685503 19898 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 20186
23:51:44.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:44.686 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.110.190:39121
23:51:44.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.687773 19898 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/logs
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/info.pb
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.190:39121
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.190
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33469
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.190.keytab
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.190
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:51:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:51:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.693506 20257 sys_catalog.cc:263] Verifying existing consensus state
23:51:44.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.698933 20257 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Bootstrap starting.
23:51:44.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.732331 20257 log.cc:824] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Log is configured to *not* fsync() on all Append() calls
23:51:44.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.736994 20257 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
23:51:44.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.737761 20257 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Bootstrap complete.
23:51:44.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.762976 20257 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:44.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.765336 20257 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
23:51:44.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.765653 20257 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3bd30c11cf524c1791f300c60a5485e4, State: Initialized, Role: FOLLOWER
23:51:44.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.766549 20257 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:44.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.771610 20261 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } } }
23:51:44.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.772239 20257 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [sys.catalog]: configured and running, proceeding with master startup.
23:51:44.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.772867 20261 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [sys.catalog]: This master's current role is: FOLLOWER
23:51:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:44.805599 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:44.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:44.805940 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.803236 20186 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111),
23:51:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:44.833279 20186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:51:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:44.857086 19405 debug-util.cc:398] Leaking SignalData structure 0x7b08000c0c40 after lost signal to thread 19337
23:51:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:44.857831 19405 debug-util.cc:398] Leaking SignalData structure 0x7b080008b720 after lost signal to thread 19408
23:51:44.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:44.865780 20211 tablet.cc:2369] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: Can't schedule compaction. Clean time has not been advanced past its initial value.
23:51:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.024730 20259 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.025517 20259 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.026276 20259 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:45.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.059665 20259 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:51:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.059944 20259 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.060334 20259 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.060637 20259 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.060953 20259 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.061262 20259 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.061480 20259 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36203/jwks.json
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.098680 20259 master_runner.cc:386] Master server non-default flags:
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.190.keytab
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.190
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.190:39121
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/info.pb
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.190
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33469
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/logs
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.100173 20259 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.101593 20259 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:45.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.142807 20280 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.182281 20261 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.182968 20261 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:45.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.190732 20206 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:51:45.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.196779 20261 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:51:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.204241 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:51:45.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.207682 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:45.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.208225 20209 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:51:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:45.209841 20261 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:51:45.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.812006 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:45.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.812448 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:46.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:46.819211 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:46.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:46.820029 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:47.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:47.327332 20292 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:47.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:47.328634 20292 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:47.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:47.370203 20292 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:51:47.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:47.383342 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:47.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:47.390584 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:51:47.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:47.391599 20206 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:51:47.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:47.392724 20292 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:51:47.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:47.825340 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:47.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:47.826056 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:45.143389 20281 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:46.536404 20279 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 20259
23:51:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.113950 20279 kernel_stack_watchdog.cc:198] Thread 20259 stuck at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:637 for 388ms:
23:51:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
23:51:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
23:51:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
23:51:48.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
23:51:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.117556 20259 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.972s user 0.001s sys 0.002s
23:51:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.117923 20259 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.972s user 0.001s sys 0.002s
23:51:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.121101 20282 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2976 milliseconds
23:51:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.123232 20284 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.123322 20259 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.125090 20259 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:48.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.128049 20259 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.129457 20259 hybrid_clock.cc:648] HybridClock initialized: now 1714780308129347 us; error 126 us; skew 500 ppm
23:51:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.130513 20259 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:48.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:48 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780308, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:48.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.137135 20259 init.cc:377] Logged in from keytab as kudu/127.19.110.190@KRBTEST.COM (short username kudu)
23:51:48.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.143416 20259 webserver.cc:460] Webserver started at http://127.19.110.190:33469/ using document root <none> and password file <none>
23:51:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.144335 20259 fs_manager.cc:359] Metadata directory not provided
23:51:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.144593 20259 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:48.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.153798 20259 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
23:51:48.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.158658 20301 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:48.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.159783 20259 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
23:51:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.160094 20259 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd384e03671b42a3a174d4d4eb4fde86"
23:51:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:38 on dist-test-slave-4816"
23:51:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.162446 20259 fs_report.cc:383] FS layout report
23:51:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:48.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/data
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:48.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.220389 20259 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.226337 20259 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.226753 20259 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:48.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.299264 20259 rpc_server.cc:307] RPC server started. Bound to: 127.19.110.190:39121
23:51:48.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.299482 20352 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.110.190:39121 every 8 connection(s)
23:51:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.302125 20259 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/data/info.pb
23:51:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.305202 19898 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 20259
23:51:48.306 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:48.307 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.110.189:44417
23:51:48.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.308974 19898 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:48.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/logs
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/info.pb
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
23:51:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.189:44417
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.189
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34575
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.189.keytab
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.189
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
23:51:48.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.315388 20353 sys_catalog.cc:263] Verifying existing consensus state
23:51:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240503 23:51:48.316202 20353 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:51:48.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.316606 20259 master.cc:537] Master@127.19.110.190:39121 shutting down...
23:51:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.375756 20259 master.cc:552] Master@127.19.110.190:39121 shutdown complete.
23:51:48.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.643790 20355 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
23:51:48.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.644338 20355 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
23:51:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.644959 20355 flags.cc:425] Enabled unsafe flag: --never_fsync=true
23:51:48.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.675599 20355 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
23:51:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.675837 20355 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
23:51:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.676096 20355 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
23:51:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.676393 20355 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
23:51:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.676602 20355 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
23:51:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.676791 20355 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
23:51:48.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.676993 20355 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36203/jwks.json
23:51:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.719520 20355 master_runner.cc:386] Master server non-default flags:
23:51:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
23:51:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.110.148:43143
23:51:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
23:51:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.110.190:39121,127.19.110.189:44417,127.19.110.188:38311
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/krb5kdc/kudu_127.19.110.189.keytab
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.110.189
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.110.189:44417
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36203/jwks.json
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/info.pb
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.110.189
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34575
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
23:51:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/logs
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1f0c0b0172fb211e74848c87fce983e45280e66d
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 May 2024 23:36:25 UTC on 1a15035a6830
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 112
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.721105 20355 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
23:51:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:48.722499 20355 file_cache.cc:492] Constructed file cache file cache with capacity 419430
23:51:48.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.739688 20362 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.832479 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.833043 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:49.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.836549 20186 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111),
23:51:49.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.836975 20186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:51:49.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:49.837833 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:49.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:49.838150 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:49.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.740413 20363 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:48.743407 20365 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
23:51:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:49.893263 20364 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
23:51:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.893316 20355 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
23:51:49.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.897181 20355 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
23:51:49.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.899693 20355 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
23:51:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.901036 20355 hybrid_clock.cc:648] HybridClock initialized: now 1714780309901005 us; error 49 us; skew 500 ppm
23:51:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.901919 20355 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
23:51:49.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:49 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780309, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:49.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.912957 20355 init.cc:377] Logged in from keytab as kudu/127.19.110.189@KRBTEST.COM (short username kudu)
23:51:49.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.918335 20355 webserver.cc:460] Webserver started at http://127.19.110.189:34575/ using document root <none> and password file <none>
23:51:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.919261 20355 fs_manager.cc:359] Metadata directory not provided
23:51:49.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.919495 20355 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
23:51:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.928516 20355 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
23:51:49.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.933876 20374 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
23:51:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.934868 20355 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
23:51:49.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.935153 20355 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data,/tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:49.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed4760ce90f144bca111b1e65d544c6a"
23:51:49.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-03 23:51:40 on dist-test-slave-4816"
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:49.937533 20355 fs_report.cc:383] FS layout report
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/data
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
23:51:49.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
23:51:50.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.001204 20355 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
23:51:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.007580 20355 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
23:51:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.007997 20355 kserver.cc:163] Server-wide thread pool size limit: 3276
23:51:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.077782 20355 rpc_server.cc:307] RPC server started. Bound to: 127.19.110.189:44417
23:51:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.077922 20425 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.19.110.189:44417 every 8 connection(s)
23:51:50.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.080466 20355 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/data/info.pb
23:51:50.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.087188 19898 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu as pid 20355
23:51:50.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
23:51:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.090662 20426 sys_catalog.cc:263] Verifying existing consensus state
23:51:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240503 23:51:50.091445 20426 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:51:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.091887 20355 master.cc:537] Master@127.19.110.189:44417 shutting down...
23:51:50.142 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.110.188:38311(127.19.110.188:38311)] connecting to peer
23:51:50.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.168025 20355 master.cc:552] Master@127.19.110.189:44417 shutdown complete.
23:51:50.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvJNS2u/test-tmp/mini-kudu-cluster247742977299241836/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
23:51:50.210 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.213 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.273 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.110.188:38311(127.19.110.188:38311)] Successfully connected to peer
23:51:50.275 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.276 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.362 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
23:51:50.370 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.110.188
23:51:50.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.399515 20432 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.399922 20432 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.403167 20432 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:51:50.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:50.418406 20206 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:51:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:50.422049 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:51:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:50.423431 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:50.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.423926 20209 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:51:50.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:50.424580 20432 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:51:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:50 dist-test-slave-4816 krb5kdc[19911](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780302, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.110.188@KRBTEST.COM
23:51:50.596 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8c25305a, L:/127.0.0.1:59990 - R:/127.19.110.188:38311] using SASL/GSSAPI
23:51:50.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.635 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.639 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.642 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.643 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.652 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.654 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.655 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.658 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.674 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.676 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.676 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.677 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.679 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.714 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.716 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.717 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.718 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.719 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.732 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.733 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.735 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.735 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.736 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.737 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.737 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.739 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.753 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.757 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.760 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.762 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.762 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:50.841593 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:50.841838 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:50.873 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.874 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.875 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.876 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.876 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.877 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.877 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.878 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.878 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:50.953 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:50.954 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:50.955 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.955 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.956 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:50.957 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.957 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.958 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:50.958 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:51.072 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:51.074 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:51.075 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:51.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:51.075 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:51.077 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:51.077 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:51.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:51.078 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:51.844647 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:51.844919 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:52.092 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:52.094 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:52.094 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:52.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:52.094 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:52.097 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:52.097 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:52.097 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:52.097 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:52.452 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:52.455 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:52.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:52.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:52.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:52.460 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:52.461 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:52.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:52.461 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:52.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:52.847628 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:52.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:52.847892 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:53.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:53.851549 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:53.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:53.851891 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.310331 20442 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.310796 20442 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:54.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.312434 20442 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:51:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:54.318172 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:51:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:54.318701 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.319038 20209 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:51:54.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.319671 20442 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:51:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:51:54 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780314, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:51:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.526482 20201 init.cc:295] Successfully reacquired a new kerberos TGT
23:51:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.840307 20186 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111),
23:51:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:54.840801 20186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:51:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:54.854796 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:54.855082 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:55.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:55.857864 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:55.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:55.858114 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:56.033 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:56.034 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:56.035 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:56.035 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:56.036 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:56.036 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:56.036 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:56.036 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:56.037 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:56.332 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:56.333 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:56.334 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:56.334 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:56.334 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:56.335 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:56.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:56.335 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:56.336 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:56.660527 19405 debug-util.cc:398] Leaking SignalData structure 0x7b08000c03a0 after lost signal to thread 19337
23:51:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:56.661643 19405 debug-util.cc:398] Leaking SignalData structure 0x7b08000c1a80 after lost signal to thread 19408
23:51:56.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:56.860780 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:56.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:56.861065 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:57.863750 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:57.864001 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:58.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:58.255 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:58.255 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:58.256 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:58.256 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:58.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:58.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:58.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:58.259 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:58.692 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:51:58.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:51:58.694 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:58.694 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:58.695 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:51:58.696 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:58.696 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:58.696 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:51:58.697 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:51:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:58.866961 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:58.867215 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:51:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:59.088232 20450 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:51:59.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:59.088635 20450 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:51:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:59.090193 20450 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:51:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:59.092247 20206 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111) [suppressed 3 similar messages]
23:51:59.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:59.094636 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:51:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:59.094995 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:51:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:59.095283 20206 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:51:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:59.095779 20450 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:51:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:59.497877 19251 debug-util.cc:398] Leaking SignalData structure 0x7b08000a99a0 after lost signal to thread 19188
23:51:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:59.498484 19251 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd7c0 after lost signal to thread 19254
23:51:59.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:59.845167 20186 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111),
23:51:59.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:51:59.845677 20186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:51:59.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:59.870043 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:51:59.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:51:59.870324 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:00.779237 20457 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:52:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:00.779647 20457 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:52:00.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:00.781342 20457 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:52:00.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:00.787937 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:52:00.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:00.788754 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:52:00.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:00.789094 20206 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:52:00.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:00.789614 20457 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:52:00.833 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:00.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:00.835 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:00.835 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:00.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:00.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:00.837 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:00.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:00.838 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:00.873186 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:00.873481 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:01.876385 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:01.876655 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:02.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:02.879458 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:02.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:02.879729 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:03.692 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:03.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:03.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:03.696 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:03.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:03.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:03.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:03.699 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:03.699 [WARN - pool-6-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:03.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:03.883029 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:03.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:03.883373 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:04.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.069793 20469 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:52:04.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.070250 20469 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:52:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.071849 20469 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:52:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:04.078406 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:52:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:04.078941 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:52:04.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.079226 20209 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:52:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.079766 20469 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:52:04.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:52:04 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780324, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:52:04.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.529114 20201 init.cc:295] Successfully reacquired a new kerberos TGT
23:52:04.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.848687 20186 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111),
23:52:04.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:04.849143 20186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:52:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:04.886319 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:04.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:04.886572 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:05.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:05.889472 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:05.889721 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:06.173 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:06.174 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:06.176 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:06.176 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:06.176 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:06.176 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:06.177 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:06.177 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:06.177 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:06.892457 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:06.892692 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:07.895354 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:07.895599 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:08.752 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:08.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:08.755 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:08.756 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:08.756 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:08.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:08.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:08.759 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:08.759 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:08.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:08.898329 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:08.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:08.898558 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:09.549679 19405 debug-util.cc:398] Leaking SignalData structure 0x7b08000c0f20 after lost signal to thread 19337
23:52:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:09.550273 19405 debug-util.cc:398] Leaking SignalData structure 0x7b080007e180 after lost signal to thread 19408
23:52:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:09.851851 20186 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111),
23:52:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:09.852249 20186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:52:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:09.901417 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:09.901659 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:10.904351 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:10.904600 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:11.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:11.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:11.294 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:11.294 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:11.295 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:11.295 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:11.295 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:11.295 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:11.296 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:11.907330 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:11.907570 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:12.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:12.854 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:12.854 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:12.854 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:12.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:12.854 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:12.854 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:12.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:12.856 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:12.911690 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:12.911934 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:13.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:13.915488 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:13.915843 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 03 23:52:14 dist-test-slave-4816 krb5kdc[19911](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714780334, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.110.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
23:52:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:14.531508 20201 init.cc:295] Successfully reacquired a new kerberos TGT
23:52:14.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:14.695 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:14.696 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:14.696 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:14.696 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:14.696 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:14.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:14.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:14.700 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:14.855443 20186 master_runner.cc:186] Error getting master registration for 127.19.110.190:39121: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111),
23:52:14.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:14.855839 20186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
23:52:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:14.918989 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:14.919232 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:15.133 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:15.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:15.134 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:15.134 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:15.134 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:15.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:15.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:15.135 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:15.137 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:15.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:15.921700 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:15.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:15.921947 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:16.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:16.351500 20490 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
23:52:16.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:16.352079 20490 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd384e03671b42a3a174d4d4eb4fde86" member_type: VOTER last_known_addr { host: "127.19.110.190" port: 39121 } } peers { permanent_uuid: "ed4760ce90f144bca111b1e65d544c6a" member_type: VOTER last_known_addr { host: "127.19.110.189" port: 44417 } } peers { permanent_uuid: "3bd30c11cf524c1791f300c60a5485e4" member_type: VOTER last_known_addr { host: "127.19.110.188" port: 38311 } }
23:52:16.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:16.353895 20490 leader_election.cc:288] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121), ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417)
23:52:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:16.356117 20206 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111) [suppressed 5 similar messages]
23:52:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:16.359040 20209 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed4760ce90f144bca111b1e65d544c6a (127.19.110.189:44417): Network error: Client connection negotiation failed: client connection to 127.19.110.189:44417: connect: Connection refused (error 111)
23:52:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:16.359508 20206 leader_election.cc:334] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fd384e03671b42a3a174d4d4eb4fde86 (127.19.110.190:39121): Network error: Client connection negotiation failed: client connection to 127.19.110.190:39121: connect: Connection refused (error 111)
23:52:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:16.359849 20206 leader_election.cc:302] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3bd30c11cf524c1791f300c60a5485e4; no voters: ed4760ce90f144bca111b1e65d544c6a, fd384e03671b42a3a174d4d4eb4fde86
23:52:16.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:16.360463 20490 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
23:52:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:16.924551 20272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3bd30c11cf524c1791f300c60a5485e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
23:52:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:16.924835 20272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
23:52:17.013 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.190:39121(127.19.110.190:39121)] connecting to peer
23:52:17.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.19.110.189:44417(127.19.110.189:44417)] connecting to peer
23:52:17.014 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:17.014 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.110.190:39121(127.19.110.190:39121)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:17.014 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:17.014 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.190:39121: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121
23:52:17.014 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.189:44417(127.19.110.189:44417)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:17.015 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.19.110.189:44417: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
23:52:17.016 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417; will retry
23:52:17.020 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26926), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [249ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [512ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [515ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [542ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [543ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [547ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [559ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [561ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [563ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [580ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [581ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [584ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [620ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [621ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [623ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [639ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [641ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [643ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [662ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [662ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [667ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [780ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [780ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [782ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [860ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [860ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [861ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [862ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [975ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [979ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [980ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [981ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [983ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [1995ms] refreshing cache from master, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [1998ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [2000ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [2001ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2001ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2355ms] refreshing cache from master, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [2358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [2362ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [2365ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2365ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [5936ms] refreshing cache from master, [5936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [5937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [5938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [5939ms] too many traces: truncated at 100 traces, deferred=Deferred@483354474(state=PENDING, result=null, callback=(continuation of Deferred@1036862473 after retry RPC after error@292528991) -> (continuation of Deferred@1126469414 after retry RPC after error@615990303) -> (continuation of Deferred@1938661772 after retry RPC after error@529830393) -> (continuation of Deferred@2024693906 after retry RPC after error@1722242084) -> (continuation of Deferred@2025330877 after retry RPC after error@1134052618) -> (continuation of Deferred@1220085531 after retry RPC after error@350452828) -> (continuation of Deferred@749138829 after retry RPC after error@756503302) -> (continuation of Deferred@756418656 after retry RPC after error@572030978) -> (continuation of Deferred@960050519 after retry RPC after error@773967381) -> (continuation of Deferred@862557881 after retry RPC after error@1427576902) -> (continuation of Deferred@636750910 after retry RPC after error@92618739) -> (continuation of Deferred@2049359268 after retry RPC after error@387862414) -> (continuation of Deferred@302920879 after retry RPC after error@1046014906) -> (continuation of Deferred@1910291016 after retry RPC after error@349956546) -> (continuation of Deferred@1997737146 after retry RPC after error@250559705) -> (continuation of Deferred@443931062 after retry RPC after error@1361819331) -> (continuation of Deferred@1051854464 after retry RPC after error@1229782139) -> (continuation of Deferred@2101359757 after retry RPC after error@1901135944) -> (continuation of Deferred@1601804004 after retry RPC after error@1903695444) -> (continuation of Deferred@232829098 after retry RPC after error@1955265046) -> (continuation of Deferred@1742183637 after retry RPC after error@1445784781) -> (continuation of Deferred@1589732162 after retry RPC after error@2100848838) -> (continuation of Deferred@651321984 after retry RPC after error@396998675) -> (continuation of Deferred@1118241495 after retry RPC after error@2087136285), errback=(continuation of Deferred@1036862473 after retry RPC after error@292528991) -> (continuation of Deferred@1126469414 after retry RPC after error@615990303) -> (continuation of Deferred@1938661772 after retry RPC after error@529830393) -> (continuation of Deferred@2024693906 after retry RPC after error@1722242084) -> (continuation of Deferred@2025330877 after retry RPC after error@1134052618) -> (continuation of Deferred@1220085531 after retry RPC after error@350452828) -> (continuation of Deferred@749138829 after retry RPC after error@756503302) -> (continuation of Deferred@756418656 after retry RPC after error@572030978) -> (continuation of Deferred@960050519 after retry RPC after error@773967381) -> (continuation of Deferred@862557881 after retry RPC after error@1427576902) -> (continuation of Deferred@636750910 after retry RPC after error@92618739) -> (continuation of Deferred@2049359268 after retry RPC after error@387862414) -> (continuation of Deferred@302920879 after retry RPC after error@1046014906) -> (continuation of Deferred@1910291016 after retry RPC after error@349956546) -> (continuation of Deferred@1997737146 after retry RPC after error@250559705) -> (continuation of Deferred@443931062 after retry RPC after error@1361819331) -> (continuation of Deferred@1051854464 after retry RPC after error@1229782139) -> (continuation of Deferred@2101359757 after retry RPC after error@1901135944) -> (continuation of Deferred@1601804004 after retry RPC after error@1903695444) -> (continuation of Deferred@232829098 after retry RPC after error@1955265046) -> (continuation of Deferred@1742183637 after retry RPC after error@1445784781) -> (continuation of Deferred@1589732162 after retry RPC after error@2100848838) -> (continuation of Deferred@651321984 after retry RPC after error@396998675) -> (continuation of Deferred@1118241495 after retry RPC after error@2087136285)))
23:52:17.025 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.110.188:38311(127.19.110.188:38311)] disconnecting while in state READY
23:52:17.025 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.110.189:44417(127.19.110.189:44417)] disconnecting while in state TERMINATED
23:52:17.026 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.110.190:39121(127.19.110.190:39121)] disconnecting while in state TERMINATED
23:52:17.030 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:52:17.031 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:52:17.032 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.110.188:38311(127.19.110.188:38311)] handling channelInactive
23:52:17.033 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.110.188:38311(127.19.110.188:38311)] cleaning up while in state READY due to: connection closed
23:52:17.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:17.034799 19898 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 20186
23:52:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T23:52:18Z chronyd exiting
23:52:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:18.306974 19251 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5440 after lost signal to thread 19188
23:52:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240503 23:52:18.307469 19251 debug-util.cc:398] Leaking SignalData structure 0x7b080009e420 after lost signal to thread 19254
23:52:19.166 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
23:52:19.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:19.167635 19175 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 19471
23:52:19.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:19.193763 19175 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 19615
23:52:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:19.219039 19175 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 19756
23:52:19.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:19.243499 19175 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 19187
23:52:19.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:19.273459 19175 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 19257
23:52:19.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240503 23:52:19.305518 19175 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvJNS2u/build/tsan/bin/kudu with pid 19332
23:52:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-03T23:52:20Z chronyd exiting
23:52:21.402 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26926), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [249ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [512ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [515ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [542ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [543ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [547ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [559ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [561ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [563ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [580ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [581ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [584ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [620ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [621ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [623ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [639ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [641ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [643ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [662ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [662ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [667ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [780ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [780ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [782ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [860ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [860ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [861ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [862ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [975ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [979ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [980ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [981ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [983ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [1995ms] refreshing cache from master, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [1998ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [2000ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [2001ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2001ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2355ms] refreshing cache from master, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [2358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [2362ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [2365ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2365ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [5936ms] refreshing cache from master, [5936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [5937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [5938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [5939ms] too many traces: truncated at 100 traces, deferred=Deferred@483354474(state=PENDING, result=null, callback=(continuation of Deferred@1036862473 after retry RPC after error@292528991) -> (continuation of Deferred@1126469414 after retry RPC after error@615990303) -> (continuation of Deferred@1938661772 after retry RPC after error@529830393) -> (continuation of Deferred@2024693906 after retry RPC after error@1722242084) -> (continuation of Deferred@2025330877 after retry RPC after error@1134052618) -> (continuation of Deferred@1220085531 after retry RPC after error@350452828) -> (continuation of Deferred@749138829 after retry RPC after error@756503302) -> (continuation of Deferred@756418656 after retry RPC after error@572030978) -> (continuation of Deferred@960050519 after retry RPC after error@773967381) -> (continuation of Deferred@862557881 after retry RPC after error@1427576902) -> (continuation of Deferred@636750910 after retry RPC after error@92618739) -> (continuation of Deferred@2049359268 after retry RPC after error@387862414) -> (continuation of Deferred@302920879 after retry RPC after error@1046014906) -> (continuation of Deferred@1910291016 after retry RPC after error@349956546) -> (continuation of Deferred@1997737146 after retry RPC after error@250559705) -> (continuation of Deferred@443931062 after retry RPC after error@1361819331) -> (continuation of Deferred@1051854464 after retry RPC after error@1229782139) -> (continuation of Deferred@2101359757 after retry RPC after error@1901135944) -> (continuation of Deferred@1601804004 after retry RPC after error@1903695444) -> (continuation of Deferred@232829098 after retry RPC after error@1955265046) -> (continuation of Deferred@1742183637 after retry RPC after error@1445784781) -> (continuation of Deferred@1589732162 after retry RPC after error@2100848838) -> (continuation of Deferred@651321984 after retry RPC after error@396998675) -> (continuation of Deferred@1118241495 after retry RPC after error@2087136285), errback=(continuation of Deferred@1036862473 after retry RPC after error@292528991) -> (continuation of Deferred@1126469414 after retry RPC after error@615990303) -> (continuation of Deferred@1938661772 after retry RPC after error@529830393) -> (continuation of Deferred@2024693906 after retry RPC after error@1722242084) -> (continuation of Deferred@2025330877 after retry RPC after error@1134052618) -> (continuation of Deferred@1220085531 after retry RPC after error@350452828) -> (continuation of Deferred@749138829 after retry RPC after error@756503302) -> (continuation of Deferred@756418656 after retry RPC after error@572030978) -> (continuation of Deferred@960050519 after retry RPC after error@773967381) -> (continuation of Deferred@862557881 after retry RPC after error@1427576902) -> (continuation of Deferred@636750910 after retry RPC after error@92618739) -> (continuation of Deferred@2049359268 after retry RPC after error@387862414) -> (continuation of Deferred@302920879 after retry RPC after error@1046014906) -> (continuation of Deferred@1910291016 after retry RPC after error@349956546) -> (continuation of Deferred@1997737146 after retry RPC after error@250559705) -> (continuation of Deferred@443931062 after retry RPC after error@1361819331) -> (continuation of Deferred@1051854464 after retry RPC after error@1229782139) -> (continuation of Deferred@2101359757 after retry RPC after error@1901135944) -> (continuation of Deferred@1601804004 after retry RPC after error@1903695444) -> (continuation of Deferred@232829098 after retry RPC after error@1955265046) -> (continuation of Deferred@1742183637 after retry RPC after error@1445784781) -> (continuation of Deferred@1589732162 after retry RPC after error@2100848838) -> (continuation of Deferred@651321984 after retry RPC after error@396998675) -> (continuation of Deferred@1118241495 after retry RPC after error@2087136285)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.110.189:44417
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26926), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [113ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [181ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [249ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [512ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [515ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [542ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [543ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [547ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [555ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [559ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [561ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [563ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [576ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [580ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [581ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [584ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [620ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [621ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [623ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [639ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [641ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [643ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [662ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [662ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [667ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [780ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [780ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [782ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [860ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [860ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [861ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [862ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [975ms] refreshing cache from master, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [979ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [980ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [981ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [983ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [1995ms] refreshing cache from master, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [1996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [1997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [1998ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [2000ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [2001ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2001ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2355ms] refreshing cache from master, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [2358ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.188:38311: OK, [2362ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.190:39121: Network error: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121, [2365ms] Sub RPC ConnectToMaster: received response from server master-127.19.110.189:44417: Network error: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [2365ms] delaying RPC due to: Service unavailable: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417, [5936ms] refreshing cache from master, [5936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.188:38311, [5937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.190:39121, [5938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.110.189:44417, [5939ms] too many traces: truncated at 100 traces, deferred=Deferred@483354474(state=PENDING, result=null, callback=(continuation of Deferred@1036862473 after retry RPC after error@292528991) -> (continuation of Deferred@1126469414 after retry RPC after error@615990303) -> (continuation of Deferred@1938661772 after retry RPC after error@529830393) -> (continuation of Deferred@2024693906 after retry RPC after error@1722242084) -> (continuation of Deferred@2025330877 after retry RPC after error@1134052618) -> (continuation of Deferred@1220085531 after retry RPC after error@350452828) -> (continuation of Deferred@749138829 after retry RPC after error@756503302) -> (continuation of Deferred@756418656 after retry RPC after error@572030978) -> (continuation of Deferred@960050519 after retry RPC after error@773967381) -> (continuation of Deferred@862557881 after retry RPC after error@1427576902) -> (continuation of Deferred@636750910 after retry RPC after error@92618739) -> (continuation of Deferred@2049359268 after retry RPC after error@387862414) -> (continuation of Deferred@302920879 after retry RPC after error@1046014906) -> (continuation of Deferred@1910291016 after retry RPC after error@349956546) -> (continuation of Deferred@1997737146 after retry RPC after error@250559705) -> (continuation of Deferred@443931062 after retry RPC after error@1361819331) -> (continuation of Deferred@1051854464 after retry RPC after error@1229782139) -> (continuation of Deferred@2101359757 after retry RPC after error@1901135944) -> (continuation of Deferred@1601804004 after retry RPC after error@1903695444) -> (continuation of Deferred@232829098 after retry RPC after error@1955265046) -> (continuation of Deferred@1742183637 after retry RPC after error@1445784781) -> (continuation of Deferred@1589732162 after retry RPC after error@2100848838) -> (continuation of Deferred@651321984 after retry RPC after error@396998675) -> (continuation of Deferred@1118241495 after retry RPC after error@2087136285), errback=(continuation of Deferred@1036862473 after retry RPC after error@292528991) -> (continuation of Deferred@1126469414 after retry RPC after error@615990303) -> (continuation of Deferred@1938661772 after retry RPC after error@529830393) -> (continuation of Deferred@2024693906 after retry RPC after error@1722242084) -> (continuation of Deferred@2025330877 after retry RPC after error@1134052618) -> (continuation of Deferred@1220085531 after retry RPC after error@350452828) -> (continuation of Deferred@749138829 after retry RPC after error@756503302) -> (continuation of Deferred@756418656 after retry RPC after error@572030978) -> (continuation of Deferred@960050519 after retry RPC after error@773967381) -> (continuation of Deferred@862557881 after retry RPC after error@1427576902) -> (continuation of Deferred@636750910 after retry RPC after error@92618739) -> (continuation of Deferred@2049359268 after retry RPC after error@387862414) -> (continuation of Deferred@302920879 after retry RPC after error@1046014906) -> (continuation of Deferred@1910291016 after retry RPC after error@349956546) -> (continuation of Deferred@1997737146 after retry RPC after error@250559705) -> (continuation of Deferred@443931062 after retry RPC after error@1361819331) -> (continuation of Deferred@1051854464 after retry RPC after error@1229782139) -> (continuation of Deferred@2101359757 after retry RPC after error@1901135944) -> (continuation of Deferred@1601804004 after retry RPC after error@1903695444) -> (continuation of Deferred@232829098 after retry RPC after error@1955265046) -> (continuation of Deferred@1742183637 after retry RPC after error@1445784781) -> (continuation of Deferred@1589732162 after retry RPC after error@2100848838) -> (continuation of Deferred@651321984 after retry RPC after error@396998675) -> (continuation of Deferred@1118241495 after retry RPC after error@2087136285)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.19.110.188:38311,127.19.110.190:39121,127.19.110.189:44417) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.190:39121(127.19.110.190:39121): Connection refused: /127.19.110.190:39121,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.19.110.189:44417(127.19.110.189:44417): Connection refused: /127.19.110.189:44417
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.19.110.189:44417
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)