08:19:24.458 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:19:24.458 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4WlLjx/test-tmp
08:19:24.458 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
08:19:24.459 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4WlLjx/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
08:19:24.460 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:19:24.776 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:19:24.776 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:19:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:24.778678 12142 mini_oidc.cc:152] Starting JWKS server
08:19:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:24.781781 12142 webserver.cc:460] Webserver started at http://127.0.0.1:38905/ using document root <none> and password file <none>
08:19:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:24.782020 12142 mini_oidc.cc:189] Starting OIDC Discovery server
08:19:24.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:24.782620 12142 webserver.cc:460] Webserver started at http://127.0.0.1:40679/ using document root <none> and password file <none>
08:19:24.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:19:24Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:19:24.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:19:24Z Disabled control of system clock
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:24.814380 12142 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/wal
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/logs
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data/info.pb
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:19:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.190:44505
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.190
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38905/jwks.json
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:19:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:19:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.135746 12153 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:19:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.136078 12153 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:19:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.136207 12153 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:19:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.143824 12153 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:19:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.143949 12153 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:19:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.144002 12153 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:19:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.144074 12153 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:19:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.144129 12153 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:19:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.144203 12153 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:19:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.144284 12153 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38905/jwks.json
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.157771 12153 master_runner.cc:386] Master server non-default flags:
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/wal
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.190:44505
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38905/jwks.json
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data/info.pb
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.190
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/logs
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:19:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:19:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:19:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:19:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:19:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
08:19:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 71
08:19:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:19:25.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.159514 12153 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:19:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.161670 12153 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:19:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.173712 12159 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.173980 12161 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.174458 12158 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.174978 12153 server_base.cc:1034] running on GCE node
08:19:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.175838 12153 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:19:25.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.177218 12153 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:19:25.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.178436 12153 hybrid_clock.cc:648] HybridClock initialized: now 1714551565178380 us; error 47 us; skew 500 ppm
08:19:25.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.178830 12153 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:19:25.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.181207 12153 webserver.cc:460] Webserver started at http://127.11.219.190:36639/ using document root <none> and password file <none>
08:19:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.181968 12153 fs_manager.cc:359] Metadata directory not provided
08:19:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.182081 12153 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:19:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.182420 12153 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:19:25.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.185046 12153 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data/instance:
08:19:25.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0"
08:19:25.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:25 on dist-test-slave-ft9l"
08:19:25.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.185829 12153 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/wal/instance:
08:19:25.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0"
08:19:25.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:25 on dist-test-slave-ft9l"
08:19:25.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.190850 12153 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.001s
08:19:25.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.194224 12167 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:19:25.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.195700 12153 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.195878 12153 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data,/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/wal
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0"
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:25 on dist-test-slave-ft9l"
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.196097 12153 fs_report.cc:383] FS layout report
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/wal
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/wal
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data/data
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:19:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:19:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.230348 12153 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:19:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.231278 12153 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:19:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.231612 12153 kserver.cc:163] Server-wide thread pool size limit: 3276
08:19:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.251442 12153 rpc_server.cc:307] RPC server started. Bound to: 127.11.219.190:44505
08:19:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.251499 12228 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.219.190:44505 every 8 connection(s)
08:19:25.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.252823 12153 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/data/info.pb
08:19:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.256492 12229 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:25.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.261555 12142 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4WlLjx/build/asan/bin/kudu as pid 12153
08:19:25.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.261798 12142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-0/wal/instance
08:19:25.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.262131 12229 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } has no permanent_uuid. Determining permanent_uuid...
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.264025 12142 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/wal
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/logs
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data/info.pb
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:19:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.189:35561
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.189
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38905/jwks.json
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:19:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:19:25.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.280416 12229 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } has no permanent_uuid. Determining permanent_uuid...
08:19:25.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.281682 12171 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.219.189:35561: connect: Connection refused (error 111)
08:19:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.283282 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.189:35561: Network error: Client connection negotiation failed: client connection to 127.11.219.189:35561: connect: Connection refused (error 111)
08:19:25.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.332718 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } attempt: 1
08:19:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.334851 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.189:35561: Network error: Client connection negotiation failed: client connection to 127.11.219.189:35561: connect: Connection refused (error 111)
08:19:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.403118 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } attempt: 2
08:19:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.405140 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.189:35561: Network error: Client connection negotiation failed: client connection to 127.11.219.189:35561: connect: Connection refused (error 111)
08:19:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.496410 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } attempt: 3
08:19:25.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.498508 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.189:35561: Network error: Client connection negotiation failed: client connection to 127.11.219.189:35561: connect: Connection refused (error 111)
08:19:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.575740 12233 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:19:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.576043 12233 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:19:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.576134 12233 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:19:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.583608 12233 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:19:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.583710 12233 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:19:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.583746 12233 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:19:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.583794 12233 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:19:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.583832 12233 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:19:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.583880 12233 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:19:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.583926 12233 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38905/jwks.json
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.592985 12233 master_runner.cc:386] Master server non-default flags:
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/wal
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.189:35561
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38905/jwks.json
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data/info.pb
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.189
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/logs
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:19:25.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:19:25.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:19:25.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
08:19:25.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 71
08:19:25.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:19:25.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.594239 12233 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:19:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.595924 12233 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:19:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.606709 12239 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.607064 12240 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.607491 12242 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.608000 12233 server_base.cc:1034] running on GCE node
08:19:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.608700 12233 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:19:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.610057 12233 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:19:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.611284 12233 hybrid_clock.cc:648] HybridClock initialized: now 1714551565611234 us; error 42 us; skew 500 ppm
08:19:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.611645 12233 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:19:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.613746 12233 webserver.cc:460] Webserver started at http://127.11.219.189:39215/ using document root <none> and password file <none>
08:19:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.614409 12233 fs_manager.cc:359] Metadata directory not provided
08:19:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.614506 12233 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:19:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.614773 12233 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:19:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.617357 12233 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data/instance:
08:19:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8de291378e94eaba67cec25238adf61"
08:19:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:25 on dist-test-slave-ft9l"
08:19:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.618113 12233 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/wal/instance:
08:19:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8de291378e94eaba67cec25238adf61"
08:19:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:25 on dist-test-slave-ft9l"
08:19:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.623099 12233 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
08:19:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.626690 12248 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.628090 12233 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.628239 12233 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data,/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/wal
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f8de291378e94eaba67cec25238adf61"
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:25 on dist-test-slave-ft9l"
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.628414 12233 fs_report.cc:383] FS layout report
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/wal
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/wal
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data/data
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:19:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:19:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:19:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.641778 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } attempt: 4
08:19:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.644752 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.189:35561: Network error: Client connection negotiation failed: client connection to 127.11.219.189:35561: connect: Connection refused (error 111)
08:19:25.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.651295 12233 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:19:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.652226 12233 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:19:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.652523 12233 kserver.cc:163] Server-wide thread pool size limit: 3276
08:19:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.673705 12233 rpc_server.cc:307] RPC server started. Bound to: 127.11.219.189:35561
08:19:25.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.673736 12309 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.219.189:35561 every 8 connection(s)
08:19:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.675067 12233 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/data/info.pb
08:19:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.678889 12310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.680723 12142 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4WlLjx/build/asan/bin/kudu as pid 12233
08:19:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.680894 12142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-1/wal/instance
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.681874 12142 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/wal
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/logs
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data/info.pb
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.188:38941
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.188
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38905/jwks.json
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:19:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:19:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.684795 12310 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } has no permanent_uuid. Determining permanent_uuid...
08:19:25.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.698053 12310 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } has no permanent_uuid. Determining permanent_uuid...
08:19:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.708401 12310 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } has no permanent_uuid. Determining permanent_uuid...
08:19:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.709627 12249 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.711364 12310 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.760780 12310 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 1
08:19:25.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.762828 12310 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.831079 12310 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 2
08:19:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.833143 12310 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:25.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.924403 12310 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 3
08:19:25.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.926343 12310 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.944008 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } attempt: 5
08:19:25.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:25.950129 12229 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } has no permanent_uuid. Determining permanent_uuid...
08:19:25.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:25.952183 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.001294 12312 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:19:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.001628 12312 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:19:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.001757 12312 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:19:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.003401 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 1
08:19:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.005162 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.009999 12312 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:19:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.010125 12312 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:19:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.010181 12312 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:19:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.010249 12312 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:19:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.010313 12312 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:19:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.010417 12312 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:19:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.010497 12312 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38905/jwks.json
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.020085 12312 master_runner.cc:386] Master server non-default flags:
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/wal
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.188:38941
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38905/jwks.json
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data/info.pb
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.188
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/logs
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 71
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:19:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.021764 12312 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:19:26.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.023578 12312 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:19:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.033991 12320 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.034471 12321 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.034960 12323 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.035073 12312 server_base.cc:1034] running on GCE node
08:19:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.035889 12312 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:19:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.037253 12312 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:19:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.038525 12312 hybrid_clock.cc:648] HybridClock initialized: now 1714551566038469 us; error 41 us; skew 500 ppm
08:19:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.038868 12312 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:19:26.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.041227 12312 webserver.cc:460] Webserver started at http://127.11.219.188:43951/ using document root <none> and password file <none>
08:19:26.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.041915 12312 fs_manager.cc:359] Metadata directory not provided
08:19:26.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.042032 12312 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:19:26.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.042330 12312 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:19:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.044867 12312 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data/instance:
08:19:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a9d4ba148bca5d1732e4557ea61"
08:19:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:26 on dist-test-slave-ft9l"
08:19:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.045651 12312 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/wal/instance:
08:19:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a9d4ba148bca5d1732e4557ea61"
08:19:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:26 on dist-test-slave-ft9l"
08:19:26.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.050616 12312 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
08:19:26.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.054061 12329 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:19:26.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.055599 12312 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.055857 12312 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data,/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/wal
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24637a9d4ba148bca5d1732e4557ea61"
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:26 on dist-test-slave-ft9l"
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.056080 12312 fs_report.cc:383] FS layout report
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/wal
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/wal
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data/data
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:19:26.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:19:26.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.069566 12310 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 4
08:19:26.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.071447 12310 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.073371 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 2
08:19:26.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.074951 12229 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:26.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.091521 12312 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:19:26.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.092343 12312 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:19:26.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.092597 12312 kserver.cc:163] Server-wide thread pool size limit: 3276
08:19:26.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.112924 12312 rpc_server.cc:307] RPC server started. Bound to: 127.11.219.188:38941
08:19:26.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.112959 12390 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.219.188:38941 every 8 connection(s)
08:19:26.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.114351 12312 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/data/info.pb
08:19:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.118186 12391 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.118266 12142 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4WlLjx/build/asan/bin/kudu as pid 12312
08:19:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.118395 12142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/master-2/wal/instance
08:19:26.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.124197 12391 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } has no permanent_uuid. Determining permanent_uuid...
08:19:26.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.137018 12391 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } has no permanent_uuid. Determining permanent_uuid...
08:19:26.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.144091 12391 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } has no permanent_uuid. Determining permanent_uuid...
08:19:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.155514 12391 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61: Bootstrap starting.
08:19:26.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.158463 12391 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61: Neither blocks nor log segments found. Creating new log.
08:19:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.159546 12391 log.cc:824] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61: Log is configured to *not* fsync() on all Append() calls
08:19:26.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.162205 12391 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61: No bootstrap required, opened a new log
08:19:26.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.168550 12391 raft_consensus.cc:348] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.168843 12391 raft_consensus.cc:374] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:26.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.168920 12391 raft_consensus.cc:729] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24637a9d4ba148bca5d1732e4557ea61, State: Initialized, Role: FOLLOWER
08:19:26.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.169445 12391 consensus_queue.cc:260] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.170248 12400 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.170540 12400 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: This master's current role is: FOLLOWER
08:19:26.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.171298 12391 sys_catalog.cc:564] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: configured and running, proceeding with master startup.
08:19:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.180006 12411 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:19:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.180119 12411 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:19:26.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.181178 12229 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 3
08:19:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.178213 12312 master_runner.cc:186] Error getting master registration for 127.11.219.190:44505: OK, instance_id { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" instance_seqno: 1714551565246973 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:19:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.186950 12312 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:19:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.191697 12229 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: Bootstrap starting.
08:19:26.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.194816 12229 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: Neither blocks nor log segments found. Creating new log.
08:19:26.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.196002 12229 log.cc:824] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: Log is configured to *not* fsync() on all Append() calls
08:19:26.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.198313 12229 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: No bootstrap required, opened a new log
08:19:26.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.205066 12229 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.205356 12229 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:26.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.205466 12229 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7e88e5b9f5f749b4b8aa13d9dba20db0, State: Initialized, Role: FOLLOWER
08:19:26.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.206086 12229 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.206967 12414 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.207279 12414 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: This master's current role is: FOLLOWER
08:19:26.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.207803 12400 raft_consensus.cc:482] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:19:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.208207 12229 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: configured and running, proceeding with master startup.
08:19:26.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.208065 12400 raft_consensus.cc:504] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.209812 12400 leader_election.cc:288] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7e88e5b9f5f749b4b8aa13d9dba20db0 (127.11.219.190:44505), f8de291378e94eaba67cec25238adf61 (127.11.219.189:35561)
08:19:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.210886 12194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24637a9d4ba148bca5d1732e4557ea61" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" is_pre_election: true
08:19:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.211479 12194 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 24637a9d4ba148bca5d1732e4557ea61 in term 0.
08:19:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.212148 12331 leader_election.cc:302] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24637a9d4ba148bca5d1732e4557ea61, 7e88e5b9f5f749b4b8aa13d9dba20db0; no voters:
08:19:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.212599 12400 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:19:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.212774 12400 raft_consensus.cc:482] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:19:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.212880 12400 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 0 FOLLOWER]: Advancing to term 1
08:19:26.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.216272 12400 raft_consensus.cc:504] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.218269 12194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24637a9d4ba148bca5d1732e4557ea61" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0"
08:19:26.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.218490 12194 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 0 FOLLOWER]: Advancing to term 1
08:19:26.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.221794 12194 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 24637a9d4ba148bca5d1732e4557ea61 in term 1.
08:19:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.222275 12331 leader_election.cc:302] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 24637a9d4ba148bca5d1732e4557ea61, 7e88e5b9f5f749b4b8aa13d9dba20db0; no voters:
08:19:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.222862 12400 leader_election.cc:288] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [CANDIDATE]: Term 1 election: Requested vote from peers 7e88e5b9f5f749b4b8aa13d9dba20db0 (127.11.219.190:44505), f8de291378e94eaba67cec25238adf61 (127.11.219.189:35561)
08:19:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.223122 12400 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 1 FOLLOWER]: Leader election won for term 1
08:19:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.223294 12400 raft_consensus.cc:686] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [term 1 LEADER]: Becoming Leader. State: Replica: 24637a9d4ba148bca5d1732e4557ea61, State: Running, Role: LEADER
08:19:26.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.224005 12400 consensus_queue.cc:237] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.234302 12173 tablet.cc:2369] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:19:26.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.243608 12423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24637a9d4ba148bca5d1732e4557ea61. Latest consensus state: current_term: 1 leader_uuid: "24637a9d4ba148bca5d1732e4557ea61" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.244081 12423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: This master's current role is: LEADER
08:19:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.246791 12426 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:19:26.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.247509 12426 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:19:26.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.230592 12153 master_runner.cc:186] Error getting master registration for 127.11.219.189:35561: OK, instance_id { permanent_uuid: "f8de291378e94eaba67cec25238adf61" instance_seqno: 1714551565668612 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:19:26.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.251199 12153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:19:26.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.280275 12429 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
08:19:26.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.282680 12429 catalog_manager.cc:1484] Initializing Kudu cluster ID...
08:19:26.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.293535 12194 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Refusing update from remote peer 24637a9d4ba148bca5d1732e4557ea61: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:19:26.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.294448 12423 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.313548 12430 mvcc.cc:205] Tried to move back new op lower bound from 7022803215526445056 to 7022803215259103232. Current Snapshot: MvccSnapshot[applied={T|T < 7022803215526445056}]
08:19:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.313426 12414 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24637a9d4ba148bca5d1732e4557ea61. Latest consensus state: current_term: 1 leader_uuid: "24637a9d4ba148bca5d1732e4557ea61" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.313714 12414 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: This master's current role is: FOLLOWER
08:19:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.323256 12431 mvcc.cc:205] Tried to move back new op lower bound from 7022803215526445056 to 7022803215259103232. Current Snapshot: MvccSnapshot[applied={T|T < 7022803215526445056}]
08:19:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.328209 12400 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24637a9d4ba148bca5d1732e4557ea61" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.328605 12400 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: This master's current role is: LEADER
08:19:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.330446 12414 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24637a9d4ba148bca5d1732e4557ea61" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.330695 12414 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: This master's current role is: FOLLOWER
08:19:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.336124 12429 catalog_manager.cc:1347] Generated new cluster ID: 6fa145b01db642519e14c73fdcd06d1b
08:19:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.336249 12429 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
08:19:26.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.352048 12429 catalog_manager.cc:1370] Generated new certificate authority record
08:19:26.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.356945 12429 catalog_manager.cc:1504] Loading token signing keys...
08:19:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.370687 12310 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } attempt: 5
08:19:26.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.384429 12310 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61: Bootstrap starting.
08:19:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.388202 12310 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61: Neither blocks nor log segments found. Creating new log.
08:19:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.391116 12310 log.cc:824] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61: Log is configured to *not* fsync() on all Append() calls
08:19:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.393222 12429 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61: Generated new TSK 0
08:19:26.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.397503 12429 catalog_manager.cc:1514] Initializing in-progress tserver states...
08:19:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.401734 12310 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61: No bootstrap required, opened a new log
08:19:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.410967 12310 raft_consensus.cc:348] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.411370 12310 raft_consensus.cc:374] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.411481 12310 raft_consensus.cc:729] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f8de291378e94eaba67cec25238adf61, State: Initialized, Role: FOLLOWER
08:19:26.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.412286 12310 consensus_queue.cc:260] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.413754 12436 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.414736 12436 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: This master's current role is: FOLLOWER
08:19:26.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.415074 12310 sys_catalog.cc:564] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: configured and running, proceeding with master startup.
08:19:26.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.415920 12273 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 0 FOLLOWER]: Advancing to term 1
08:19:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.415139 12275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24637a9d4ba148bca5d1732e4557ea61" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f8de291378e94eaba67cec25238adf61" is_pre_election: true
08:19:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.415422 12274 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24637a9d4ba148bca5d1732e4557ea61" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f8de291378e94eaba67cec25238adf61"
08:19:26.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.420576 12273 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 1 FOLLOWER]: Refusing update from remote peer 24637a9d4ba148bca5d1732e4557ea61: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.422811 12423 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.426467 12142 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/wal
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/logs
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data/info.pb
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:19:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.129:0
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.219.129
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.129
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:19:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:19:26.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.480209 12436 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 24637a9d4ba148bca5d1732e4557ea61. Latest consensus state: current_term: 1 leader_uuid: "24637a9d4ba148bca5d1732e4557ea61" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.480518 12436 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: This master's current role is: FOLLOWER
08:19:26.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.480995 12444 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "24637a9d4ba148bca5d1732e4557ea61" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.481274 12444 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: This master's current role is: FOLLOWER
08:19:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.530339 12423 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "24637a9d4ba148bca5d1732e4557ea61" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.530686 12423 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24637a9d4ba148bca5d1732e4557ea61 [sys.catalog]: This master's current role is: LEADER
08:19:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.538460 12454 catalog_manager.cc:1259] Loaded cluster ID: 6fa145b01db642519e14c73fdcd06d1b
08:19:26.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.538594 12454 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61: loading cluster ID for follower catalog manager: success
08:19:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.542521 12454 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61: acquiring CA information for follower catalog manager: success
08:19:26.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.549932 12454 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:19:26.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.902156 12440 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:19:26.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.902444 12440 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:19:26.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.902606 12440 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:19:26.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.910245 12440 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:19:26.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.910427 12440 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:19:26.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.910482 12440 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:19:26.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.910544 12440 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:19:26.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.910646 12440 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.219.129
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.919998 12440 tablet_server_runner.cc:78] Tablet server non-default flags:
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/wal
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.129:0
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data/info.pb
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.129
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.219.129
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/logs
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 71
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:19:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.921602 12440 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:19:26.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.923497 12440 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:19:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.936882 12461 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.936853 12460 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:26.937523 12463 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:26.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.937943 12440 server_base.cc:1034] running on GCE node
08:19:26.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.938581 12440 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:19:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.939770 12440 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:19:26.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.941001 12440 hybrid_clock.cc:648] HybridClock initialized: now 1714551566940943 us; error 74 us; skew 500 ppm
08:19:26.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.941411 12440 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:19:26.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.949164 12440 webserver.cc:460] Webserver started at http://127.11.219.129:33903/ using document root <none> and password file <none>
08:19:26.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.950245 12440 fs_manager.cc:359] Metadata directory not provided
08:19:26.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.950409 12440 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:19:26.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.950830 12440 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:19:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.954640 12440 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data/instance:
08:19:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eccd08210d84122b76190a299fa7373"
08:19:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:26 on dist-test-slave-ft9l"
08:19:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.955736 12440 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/wal/instance:
08:19:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eccd08210d84122b76190a299fa7373"
08:19:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:26 on dist-test-slave-ft9l"
08:19:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.962985 12440 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
08:19:26.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.967728 12469 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:19:26.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.969560 12440 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.969769 12440 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data,/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/wal
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eccd08210d84122b76190a299fa7373"
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:26 on dist-test-slave-ft9l"
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:26.970037 12440 fs_report.cc:383] FS layout report
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/wal
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/wal
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data/data
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:19:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:19:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.014727 12440 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:19:27.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.015841 12440 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:19:27.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.016131 12440 kserver.cc:163] Server-wide thread pool size limit: 3276
08:19:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.019649 12440 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:19:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.019752 12440 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:19:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.019870 12440 ts_tablet_manager.cc:590] Registered 0 tablets
08:19:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.019927 12440 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:19:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.122467 12440 rpc_server.cc:307] RPC server started. Bound to: 127.11.219.129:33665
08:19:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.122637 12593 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.219.129:33665 every 8 connection(s)
08:19:27.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.124075 12440 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/data/info.pb
08:19:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.127768 12595 heartbeater.cc:347] Connected to a master server at 127.11.219.190:44505
08:19:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.128011 12595 heartbeater.cc:464] Registering TS with master...
08:19:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.128921 12595 heartbeater.cc:511] Master 127.11.219.190:44505 requested a full tablet report, sending...
08:19:27.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.133285 12184 ts_manager.cc:194] Registered new tserver with Master: 4eccd08210d84122b76190a299fa7373 (127.11.219.129:33665)
08:19:27.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.133684 12142 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4WlLjx/build/asan/bin/kudu as pid 12440
08:19:27.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.133905 12142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-0/wal/instance
08:19:27.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.137143 12594 heartbeater.cc:347] Connected to a master server at 127.11.219.188:38941
08:19:27.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.137290 12594 heartbeater.cc:464] Registering TS with master...
08:19:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.137866 12594 heartbeater.cc:511] Master 127.11.219.188:38941 requested a full tablet report, sending...
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.139227 12142 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/wal
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/logs
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data/info.pb
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.130:0
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.219.130
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.130
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:19:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:19:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:19:27.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.140174 12346 ts_manager.cc:194] Registered new tserver with Master: 4eccd08210d84122b76190a299fa7373 (127.11.219.129:33665)
08:19:27.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.143278 12346 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.219.129:34843
08:19:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.148347 12596 heartbeater.cc:347] Connected to a master server at 127.11.219.189:35561
08:19:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.148475 12596 heartbeater.cc:464] Registering TS with master...
08:19:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.148782 12596 heartbeater.cc:511] Master 127.11.219.189:35561 requested a full tablet report, sending...
08:19:27.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.150276 12265 ts_manager.cc:194] Registered new tserver with Master: 4eccd08210d84122b76190a299fa7373 (127.11.219.129:33665)
08:19:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.249900 12426 catalog_manager.cc:1259] Loaded cluster ID: 6fa145b01db642519e14c73fdcd06d1b
08:19:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.250000 12426 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: loading cluster ID for follower catalog manager: success
08:19:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.253109 12426 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: acquiring CA information for follower catalog manager: success
08:19:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.254940 12426 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:19:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.464349 12598 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:19:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.464619 12598 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:19:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.464733 12598 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:19:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.472064 12598 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:19:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.472208 12598 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:19:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.472245 12598 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:19:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.472288 12598 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:19:27.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.472352 12598 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.219.130
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.481464 12598 tablet_server_runner.cc:78] Tablet server non-default flags:
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/wal
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.130:0
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data/info.pb
08:19:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.130
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.219.130
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/logs
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 71
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:19:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.482697 12598 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:19:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.484563 12598 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:19:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.496799 12604 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.497277 12605 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.497638 12607 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.498010 12598 server_base.cc:1034] running on GCE node
08:19:27.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.498625 12598 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:19:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.499584 12598 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:19:27.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.500802 12598 hybrid_clock.cc:648] HybridClock initialized: now 1714551567500742 us; error 50 us; skew 500 ppm
08:19:27.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.501138 12598 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:19:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.503355 12598 webserver.cc:460] Webserver started at http://127.11.219.130:39633/ using document root <none> and password file <none>
08:19:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.504047 12598 fs_manager.cc:359] Metadata directory not provided
08:19:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.504140 12598 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:19:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.504393 12598 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:19:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.506870 12598 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data/instance:
08:19:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a4f0cce1da9425a9bb0b9566d2aef09"
08:19:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:27 on dist-test-slave-ft9l"
08:19:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.507639 12598 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/wal/instance:
08:19:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a4f0cce1da9425a9bb0b9566d2aef09"
08:19:27.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:27 on dist-test-slave-ft9l"
08:19:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.512501 12598 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
08:19:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.515851 12613 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.517233 12598 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.517381 12598 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data,/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/wal
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a4f0cce1da9425a9bb0b9566d2aef09"
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:27 on dist-test-slave-ft9l"
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.517588 12598 fs_report.cc:383] FS layout report
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/wal
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/wal
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data/data
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:19:27.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:19:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:19:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.544824 12598 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:19:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.545686 12598 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:19:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.545974 12598 kserver.cc:163] Server-wide thread pool size limit: 3276
08:19:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.548918 12598 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:19:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.549149 12598 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:19:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.549448 12598 ts_tablet_manager.cc:590] Registered 0 tablets
08:19:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.549520 12598 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:19:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.610942 12598 rpc_server.cc:307] RPC server started. Bound to: 127.11.219.130:40423
08:19:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.610991 12734 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.219.130:40423 every 8 connection(s)
08:19:27.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.612290 12598 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/data/info.pb
08:19:27.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.614892 12736 heartbeater.cc:347] Connected to a master server at 127.11.219.190:44505
08:19:27.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.615316 12736 heartbeater.cc:464] Registering TS with master...
08:19:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.616688 12736 heartbeater.cc:511] Master 127.11.219.190:44505 requested a full tablet report, sending...
08:19:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.618680 12184 ts_manager.cc:194] Registered new tserver with Master: 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423)
08:19:27.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.619611 12142 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4WlLjx/build/asan/bin/kudu as pid 12598
08:19:27.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.619773 12142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-1/wal/instance
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.623998 12142 external_mini_cluster.cc:1326] Running /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4WlLjx/build/asan/bin/kudu
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/wal
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/logs
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data/info.pb
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.131:0
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.219.131
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.131
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:19:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:19:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:19:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.626319 12735 heartbeater.cc:347] Connected to a master server at 127.11.219.188:38941
08:19:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.626570 12735 heartbeater.cc:464] Registering TS with master...
08:19:27.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.627177 12735 heartbeater.cc:511] Master 127.11.219.188:38941 requested a full tablet report, sending...
08:19:27.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.628190 12346 ts_manager.cc:194] Registered new tserver with Master: 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423)
08:19:27.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.628998 12346 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.219.130:34419
08:19:27.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.630883 12737 heartbeater.cc:347] Connected to a master server at 127.11.219.189:35561
08:19:27.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.631021 12737 heartbeater.cc:464] Registering TS with master...
08:19:27.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.631364 12737 heartbeater.cc:511] Master 127.11.219.189:35561 requested a full tablet report, sending...
08:19:27.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.632314 12265 ts_manager.cc:194] Registered new tserver with Master: 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423)
08:19:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.949415 12740 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:19:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.949723 12740 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:19:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.949884 12740 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:19:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.957492 12740 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:19:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.957686 12740 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:19:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.957743 12740 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:19:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.957804 12740 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:19:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.957906 12740 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.219.131
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.967238 12740 tablet_server_runner.cc:78] Tablet server non-default flags:
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.219.148:34899
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/wal
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.219.131:0
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data/info.pb
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.219.131
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:19:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.219.190:44505,127.11.219.189:35561,127.11.219.188:38941
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.219.131
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/logs
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 May 2024 07:43:24 UTC on efd1337a2aa3
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 71
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:19:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.968798 12740 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:19:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.970504 12740 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:19:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.982764 12746 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.983217 12745 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:27.983625 12748 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:19:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.983817 12740 server_base.cc:1034] running on GCE node
08:19:27.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.984428 12740 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:19:27.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.985399 12740 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:19:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.986635 12740 hybrid_clock.cc:648] HybridClock initialized: now 1714551567986596 us; error 26 us; skew 500 ppm
08:19:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.986966 12740 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:19:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.989308 12740 webserver.cc:460] Webserver started at http://127.11.219.131:44121/ using document root <none> and password file <none>
08:19:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.990036 12740 fs_manager.cc:359] Metadata directory not provided
08:19:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.990149 12740 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:19:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.990446 12740 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:19:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.992991 12740 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data/instance:
08:19:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a78b96a30bad4c56ba1dc819a7725729"
08:19:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:27 on dist-test-slave-ft9l"
08:19:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.993799 12740 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/wal/instance:
08:19:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a78b96a30bad4c56ba1dc819a7725729"
08:19:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:27 on dist-test-slave-ft9l"
08:19:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:27.998867 12740 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
08:19:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.002548 12754 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.003993 12740 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.004163 12740 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data,/tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/wal
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a78b96a30bad4c56ba1dc819a7725729"
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-01 08:19:27 on dist-test-slave-ft9l"
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.004379 12740 fs_report.cc:383] FS layout report
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/wal
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/wal
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data/data
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:19:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:19:28.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:19:28.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.035668 12740 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:19:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.036708 12740 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:19:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.036998 12740 kserver.cc:163] Server-wide thread pool size limit: 3276
08:19:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.040024 12740 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:19:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.040227 12740 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:19:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.040396 12740 ts_tablet_manager.cc:590] Registered 0 tablets
08:19:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.040787 12740 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
08:19:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.096140 12740 rpc_server.cc:307] RPC server started. Bound to: 127.11.219.131:41169
08:19:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.096176 12875 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.219.131:41169 every 8 connection(s)
08:19:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.097407 12740 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/data/info.pb
08:19:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.100032 12877 heartbeater.cc:347] Connected to a master server at 127.11.219.190:44505
08:19:28.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.100333 12877 heartbeater.cc:464] Registering TS with master...
08:19:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.101509 12877 heartbeater.cc:511] Master 127.11.219.190:44505 requested a full tablet report, sending...
08:19:28.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.103869 12142 external_mini_cluster.cc:1388] Started /tmp/dist-test-task4WlLjx/build/asan/bin/kudu as pid 12740
08:19:28.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.104053 12142 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task4WlLjx/test-tmp/mini-kudu-cluster8592827096567382011/ts-2/wal/instance
08:19:28.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.106249 12184 ts_manager.cc:194] Registered new tserver with Master: a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.107017 12876 heartbeater.cc:347] Connected to a master server at 127.11.219.188:38941
08:19:28.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.107151 12876 heartbeater.cc:464] Registering TS with master...
08:19:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.107683 12876 heartbeater.cc:511] Master 127.11.219.188:38941 requested a full tablet report, sending...
08:19:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.108790 12346 ts_manager.cc:194] Registered new tserver with Master: a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.109616 12878 heartbeater.cc:347] Connected to a master server at 127.11.219.189:35561
08:19:28.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.109755 12878 heartbeater.cc:464] Registering TS with master...
08:19:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.109925 12346 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.219.131:38077
08:19:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.110658 12878 heartbeater.cc:511] Master 127.11.219.189:35561 requested a full tablet report, sending...
08:19:28.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.111760 12265 ts_manager.cc:194] Registered new tserver with Master: a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.121232 12142 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
08:19:28.123 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:19:28.123 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:19:28.124 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.219.190"
port: 44505
}
bound_http_address {
host: "127.11.219.190"
port: 36639
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.219.189"
port: 35561
}
bound_http_address {
host: "127.11.219.189"
port: 39215
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.219.188"
port: 38941
}
bound_http_address {
host: "127.11.219.188"
port: 43951
}
}
}
08:19:28.124 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:19:28.125 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.219.129"
port: 33665
}
bound_http_address {
host: "127.11.219.129"
port: 33903
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.219.130"
port: 40423
}
bound_http_address {
host: "127.11.219.130"
port: 39633
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.219.131"
port: 41169
}
bound_http_address {
host: "127.11.219.131"
port: 44121
}
}
}
08:19:28.125 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:19:28.127 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:19:28.129 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.219.189:35561(127.11.219.189:35561)] connecting to peer
08:19:28.129 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:28.130 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.219.190:44505(127.11.219.190:44505)] connecting to peer
08:19:28.130 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.219.189:35561(127.11.219.189:35561)] Successfully connected to peer
08:19:28.131 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.11.219.188:38941(127.11.219.188:38941)] Successfully connected to peer
08:19:28.133 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.11.219.188
08:19:28.133 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.11.219.189
08:19:28.138 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.219.190:44505(127.11.219.190:44505)] Successfully connected to peer
08:19:28.146 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.11.219.190
08:19:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.148414 12594 heartbeater.cc:503] Master 127.11.219.188:38941 was elected leader, sending a full tablet report...
08:19:28.150 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x81685980, L:/127.0.0.1:33394 - R:/127.11.219.188:38941] using SASL/PLAIN
08:19:28.154 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:19:28.154 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1b1dacb6, L:/127.0.0.1:60012 - R:/127.11.219.189:35561] using SASL/PLAIN
08:19:28.154 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=26), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [24ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: OK, deferred=null) after lookup
08:19:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.160377 12345 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33394:
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
08:19:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:28.162943 12345 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:19:28.165 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8fa8c125, L:/127.0.0.1:46680 - R:/127.11.219.190:44505] using SASL/PLAIN
08:19:28.186 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 6d150404b8dc42f6857dff861e28d5e8
08:19:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.231316 12528 tablet_service.cc:1467] Processing CreateTablet for tablet bcdb3b61c8dc43399f125e655150f8b8 (DEFAULT_TABLE table=txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:19:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.231562 12666 tablet_service.cc:1467] Processing CreateTablet for tablet bcdb3b61c8dc43399f125e655150f8b8 (DEFAULT_TABLE table=txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:19:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.231348 12670 tablet_service.cc:1467] Processing CreateTablet for tablet 02c80bdaca9e44b196d252a862b98a0b (DEFAULT_TABLE table=txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:19:28.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.232914 12666 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bcdb3b61c8dc43399f125e655150f8b8. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.232941 12528 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bcdb3b61c8dc43399f125e655150f8b8. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.233747 12806 tablet_service.cc:1467] Processing CreateTablet for tablet bcdb3b61c8dc43399f125e655150f8b8 (DEFAULT_TABLE table=txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:19:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.234462 12529 tablet_service.cc:1467] Processing CreateTablet for tablet 02c80bdaca9e44b196d252a862b98a0b (DEFAULT_TABLE table=txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:19:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.234941 12529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02c80bdaca9e44b196d252a862b98a0b. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.235214 12806 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bcdb3b61c8dc43399f125e655150f8b8. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.235707 12670 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02c80bdaca9e44b196d252a862b98a0b. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.236732 12808 tablet_service.cc:1467] Processing CreateTablet for tablet 02c80bdaca9e44b196d252a862b98a0b (DEFAULT_TABLE table=txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:19:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.237211 12808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02c80bdaca9e44b196d252a862b98a0b. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.252033 12901 tablet_bootstrap.cc:492] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373: Bootstrap starting.
08:19:28.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.256708 12901 tablet_bootstrap.cc:654] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373: Neither blocks nor log segments found. Creating new log.
08:19:28.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.258260 12901 log.cc:824] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373: Log is configured to *not* fsync() on all Append() calls
08:19:28.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.260928 12902 tablet_bootstrap.cc:492] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09: Bootstrap starting.
08:19:28.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.263854 12901 tablet_bootstrap.cc:492] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373: No bootstrap required, opened a new log
08:19:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.264314 12901 ts_tablet_manager.cc:1367] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373: Time spent bootstrapping tablet: real 0.013s user 0.010s sys 0.000s
08:19:28.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.264535 12902 tablet_bootstrap.cc:654] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09: Neither blocks nor log segments found. Creating new log.
08:19:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.265758 12902 log.cc:824] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09: Log is configured to *not* fsync() on all Append() calls
08:19:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.268271 12903 tablet_bootstrap.cc:492] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729: Bootstrap starting.
08:19:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.270391 12902 tablet_bootstrap.cc:492] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09: No bootstrap required, opened a new log
08:19:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.270781 12902 ts_tablet_manager.cc:1367] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.004s
08:19:28.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.273698 12903 tablet_bootstrap.cc:654] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729: Neither blocks nor log segments found. Creating new log.
08:19:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.274972 12903 log.cc:824] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729: Log is configured to *not* fsync() on all Append() calls
08:19:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.275365 12901 raft_consensus.cc:348] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.275858 12901 raft_consensus.cc:374] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.275978 12901 raft_consensus.cc:729] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4eccd08210d84122b76190a299fa7373, State: Initialized, Role: FOLLOWER
08:19:28.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.276741 12901 consensus_queue.cc:260] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.277839 12903 tablet_bootstrap.cc:492] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729: No bootstrap required, opened a new log
08:19:28.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.278123 12903 ts_tablet_manager.cc:1367] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.002s
08:19:28.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.278472 12901 ts_tablet_manager.cc:1398] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373: Time spent starting tablet: real 0.014s user 0.008s sys 0.004s
08:19:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.279896 12901 tablet_bootstrap.cc:492] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373: Bootstrap starting.
08:19:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.279771 12902 raft_consensus.cc:348] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.280176 12902 raft_consensus.cc:374] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.280272 12902 raft_consensus.cc:729] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a4f0cce1da9425a9bb0b9566d2aef09, State: Initialized, Role: FOLLOWER
08:19:28.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.280905 12902 consensus_queue.cc:260] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.281688 12901 tablet_bootstrap.cc:654] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373: Neither blocks nor log segments found. Creating new log.
08:19:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.285977 12901 tablet_bootstrap.cc:492] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373: No bootstrap required, opened a new log
08:19:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.286137 12901 ts_tablet_manager.cc:1367] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.004s
08:19:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.287148 12901 raft_consensus.cc:348] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.287426 12901 raft_consensus.cc:374] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.287514 12901 raft_consensus.cc:729] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4eccd08210d84122b76190a299fa7373, State: Initialized, Role: FOLLOWER
08:19:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.287889 12901 consensus_queue.cc:260] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.289489 12902 ts_tablet_manager.cc:1398] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09: Time spent starting tablet: real 0.019s user 0.010s sys 0.007s
08:19:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.290046 12902 tablet_bootstrap.cc:492] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09: Bootstrap starting.
08:19:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.289575 12735 heartbeater.cc:503] Master 127.11.219.188:38941 was elected leader, sending a full tablet report...
08:19:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.291676 12901 ts_tablet_manager.cc:1398] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373: Time spent starting tablet: real 0.005s user 0.003s sys 0.000s
08:19:28.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.291870 12902 tablet_bootstrap.cc:654] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09: Neither blocks nor log segments found. Creating new log.
08:19:28.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.294080 12902 tablet_bootstrap.cc:492] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09: No bootstrap required, opened a new log
08:19:28.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.295360 12902 ts_tablet_manager.cc:1367] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
08:19:28.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.296344 12902 raft_consensus.cc:348] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.296576 12902 raft_consensus.cc:374] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.296674 12902 raft_consensus.cc:729] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a4f0cce1da9425a9bb0b9566d2aef09, State: Initialized, Role: FOLLOWER
08:19:28.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.296965 12902 consensus_queue.cc:260] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.297611 12902 ts_tablet_manager.cc:1398] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
08:19:28.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.301836 12903 raft_consensus.cc:348] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.302227 12903 raft_consensus.cc:374] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.302328 12903 raft_consensus.cc:729] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a78b96a30bad4c56ba1dc819a7725729, State: Initialized, Role: FOLLOWER
08:19:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.302946 12903 consensus_queue.cc:260] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.304035 12876 heartbeater.cc:503] Master 127.11.219.188:38941 was elected leader, sending a full tablet report...
08:19:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.304615 12903 ts_tablet_manager.cc:1398] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729: Time spent starting tablet: real 0.026s user 0.005s sys 0.006s
08:19:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.304973 12903 tablet_bootstrap.cc:492] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729: Bootstrap starting.
08:19:28.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.307473 12903 tablet_bootstrap.cc:654] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729: Neither blocks nor log segments found. Creating new log.
08:19:28.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.309481 12903 tablet_bootstrap.cc:492] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729: No bootstrap required, opened a new log
08:19:28.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.309600 12903 ts_tablet_manager.cc:1367] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.001s
08:19:28.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.310171 12903 raft_consensus.cc:348] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.310313 12903 raft_consensus.cc:374] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.310369 12903 raft_consensus.cc:729] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a78b96a30bad4c56ba1dc819a7725729, State: Initialized, Role: FOLLOWER
08:19:28.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.310525 12903 consensus_queue.cc:260] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.310923 12903 ts_tablet_manager.cc:1398] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
08:19:28.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:28.350685 12879 tablet.cc:2369] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:19:28.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:28.370922 12738 tablet.cc:2369] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:19:28.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:28.390233 12597 tablet.cc:2369] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:19:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.426268 12907 raft_consensus.cc:482] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:19:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.426528 12907 raft_consensus.cc:504] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.428292 12907 leader_election.cc:288] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423), a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.435190 12690 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02c80bdaca9e44b196d252a862b98a0b" candidate_uuid: "4eccd08210d84122b76190a299fa7373" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" is_pre_election: true
08:19:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.435381 12831 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02c80bdaca9e44b196d252a862b98a0b" candidate_uuid: "4eccd08210d84122b76190a299fa7373" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a78b96a30bad4c56ba1dc819a7725729" is_pre_election: true
08:19:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.435704 12690 raft_consensus.cc:2454] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4eccd08210d84122b76190a299fa7373 in term 0.
08:19:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.435978 12831 raft_consensus.cc:2454] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4eccd08210d84122b76190a299fa7373 in term 0.
08:19:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.436314 12473 leader_election.cc:302] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4eccd08210d84122b76190a299fa7373, 9a4f0cce1da9425a9bb0b9566d2aef09; no voters:
08:19:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.436733 12907 raft_consensus.cc:2781] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:19:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.436866 12907 raft_consensus.cc:482] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:19:28.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.436986 12907 raft_consensus.cc:3037] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.438925 12907 raft_consensus.cc:504] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.439559 12907 leader_election.cc:288] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [CANDIDATE]: Term 1 election: Requested vote from peers 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423), a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.439996 12690 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02c80bdaca9e44b196d252a862b98a0b" candidate_uuid: "4eccd08210d84122b76190a299fa7373" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09"
08:19:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.440028 12831 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02c80bdaca9e44b196d252a862b98a0b" candidate_uuid: "4eccd08210d84122b76190a299fa7373" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a78b96a30bad4c56ba1dc819a7725729"
08:19:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.440156 12690 raft_consensus.cc:3037] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.440196 12831 raft_consensus.cc:3037] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.442080 12690 raft_consensus.cc:2454] T 02c80bdaca9e44b196d252a862b98a0b P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4eccd08210d84122b76190a299fa7373 in term 1.
08:19:28.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.442173 12831 raft_consensus.cc:2454] T 02c80bdaca9e44b196d252a862b98a0b P a78b96a30bad4c56ba1dc819a7725729 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4eccd08210d84122b76190a299fa7373 in term 1.
08:19:28.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.442473 12473 leader_election.cc:302] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4eccd08210d84122b76190a299fa7373, 9a4f0cce1da9425a9bb0b9566d2aef09; no voters:
08:19:28.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.442790 12907 raft_consensus.cc:2781] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 1 FOLLOWER]: Leader election won for term 1
08:19:28.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.443285 12907 raft_consensus.cc:686] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [term 1 LEADER]: Becoming Leader. State: Replica: 4eccd08210d84122b76190a299fa7373, State: Running, Role: LEADER
08:19:28.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.443888 12907 consensus_queue.cc:237] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.449064 12346 catalog_manager.cc:5484] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4eccd08210d84122b76190a299fa7373 (127.11.219.129). New cstate: current_term: 1 leader_uuid: "4eccd08210d84122b76190a299fa7373" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } health_report { overall_health: UNKNOWN } } }
08:19:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.522809 12908 raft_consensus.cc:482] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:19:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.523005 12908 raft_consensus.cc:504] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.524928 12908 leader_election.cc:288] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4eccd08210d84122b76190a299fa7373 (127.11.219.129:33665), a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.531769 12831 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bcdb3b61c8dc43399f125e655150f8b8" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a78b96a30bad4c56ba1dc819a7725729" is_pre_election: true
08:19:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.531997 12831 raft_consensus.cc:2454] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 0.
08:19:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.531927 12549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bcdb3b61c8dc43399f125e655150f8b8" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4eccd08210d84122b76190a299fa7373" is_pre_election: true
08:19:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.532377 12549 raft_consensus.cc:2454] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 0.
08:19:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.532442 12615 leader_election.cc:302] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9a4f0cce1da9425a9bb0b9566d2aef09, a78b96a30bad4c56ba1dc819a7725729; no voters:
08:19:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.532918 12908 raft_consensus.cc:2781] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:19:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.533035 12908 raft_consensus.cc:482] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:19:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.533097 12908 raft_consensus.cc:3037] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.535372 12908 raft_consensus.cc:504] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.535987 12908 leader_election.cc:288] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 election: Requested vote from peers 4eccd08210d84122b76190a299fa7373 (127.11.219.129:33665), a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.536432 12549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bcdb3b61c8dc43399f125e655150f8b8" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4eccd08210d84122b76190a299fa7373"
08:19:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.536598 12549 raft_consensus.cc:3037] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.536574 12831 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bcdb3b61c8dc43399f125e655150f8b8" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a78b96a30bad4c56ba1dc819a7725729"
08:19:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.536839 12831 raft_consensus.cc:3037] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.538494 12549 raft_consensus.cc:2454] T bcdb3b61c8dc43399f125e655150f8b8 P 4eccd08210d84122b76190a299fa7373 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 1.
08:19:28.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.538738 12831 raft_consensus.cc:2454] T bcdb3b61c8dc43399f125e655150f8b8 P a78b96a30bad4c56ba1dc819a7725729 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 1.
08:19:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.538918 12615 leader_election.cc:302] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4eccd08210d84122b76190a299fa7373, 9a4f0cce1da9425a9bb0b9566d2aef09; no voters:
08:19:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.539220 12908 raft_consensus.cc:2781] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 1 FOLLOWER]: Leader election won for term 1
08:19:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.539700 12908 raft_consensus.cc:686] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 1 LEADER]: Becoming Leader. State: Replica: 9a4f0cce1da9425a9bb0b9566d2aef09, State: Running, Role: LEADER
08:19:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.540423 12908 consensus_queue.cc:237] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.545375 12346 catalog_manager.cc:5484] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130). New cstate: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } health_report { overall_health: UNKNOWN } } }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.654592 12346 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33400:
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:19:28.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:28.655645 12346 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:19:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.666962 12529 tablet_service.cc:1467] Processing CreateTablet for tablet 77ad290ad86146fc9f70a7dbdaafe667 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b33d8447e8894ff29b1664edcf590b7b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:19:28.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.667279 12670 tablet_service.cc:1467] Processing CreateTablet for tablet 77ad290ad86146fc9f70a7dbdaafe667 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b33d8447e8894ff29b1664edcf590b7b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:19:28.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.668076 12529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ad290ad86146fc9f70a7dbdaafe667. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.668076 12670 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ad290ad86146fc9f70a7dbdaafe667. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.668452 12808 tablet_service.cc:1467] Processing CreateTablet for tablet 77ad290ad86146fc9f70a7dbdaafe667 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=b33d8447e8894ff29b1664edcf590b7b]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:19:28.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.669097 12808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 77ad290ad86146fc9f70a7dbdaafe667. 1 dirs total, 0 dirs full, 0 dirs failed
08:19:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.674878 12902 tablet_bootstrap.cc:492] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: Bootstrap starting.
08:19:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.676369 12902 tablet_bootstrap.cc:654] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: Neither blocks nor log segments found. Creating new log.
08:19:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.676903 12903 tablet_bootstrap.cc:492] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: Bootstrap starting.
08:19:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.678303 12903 tablet_bootstrap.cc:654] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: Neither blocks nor log segments found. Creating new log.
08:19:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.678314 12902 tablet_bootstrap.cc:492] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: No bootstrap required, opened a new log
08:19:28.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.678478 12902 ts_tablet_manager.cc:1367] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
08:19:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.679144 12902 raft_consensus.cc:348] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.679402 12902 raft_consensus.cc:374] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.679476 12902 raft_consensus.cc:729] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a4f0cce1da9425a9bb0b9566d2aef09, State: Initialized, Role: FOLLOWER
08:19:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.679798 12902 consensus_queue.cc:260] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.680245 12908 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.680471 12908 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: This TxnStatusTablet replica's current role is: FOLLOWER
08:19:28.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.682523 12902 ts_tablet_manager.cc:1398] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: Time spent starting tablet: real 0.004s user 0.000s sys 0.003s
08:19:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.683656 12903 tablet_bootstrap.cc:492] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: No bootstrap required, opened a new log
08:19:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.683836 12903 ts_tablet_manager.cc:1367] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.001s
08:19:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.684604 12903 raft_consensus.cc:348] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.684830 12903 raft_consensus.cc:374] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.684904 12903 raft_consensus.cc:729] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a78b96a30bad4c56ba1dc819a7725729, State: Initialized, Role: FOLLOWER
08:19:28.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.685540 12903 consensus_queue.cc:260] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.686358 12903 ts_tablet_manager.cc:1398] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
08:19:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.686306 12911 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.686595 12911 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: This TxnStatusTablet replica's current role is: FOLLOWER
08:19:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.688449 12901 tablet_bootstrap.cc:492] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: Bootstrap starting.
08:19:28.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.690093 12901 tablet_bootstrap.cc:654] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: Neither blocks nor log segments found. Creating new log.
08:19:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.696318 12901 tablet_bootstrap.cc:492] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: No bootstrap required, opened a new log
08:19:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.696516 12901 ts_tablet_manager.cc:1367] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: Time spent bootstrapping tablet: real 0.008s user 0.007s sys 0.000s
08:19:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.697135 12901 raft_consensus.cc:348] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.697341 12901 raft_consensus.cc:374] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:19:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.697407 12901 raft_consensus.cc:729] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4eccd08210d84122b76190a299fa7373, State: Initialized, Role: FOLLOWER
08:19:28.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.697669 12901 consensus_queue.cc:260] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.698230 12901 ts_tablet_manager.cc:1398] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
08:19:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.698098 12907 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.698388 12907 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: This TxnStatusTablet replica's current role is: FOLLOWER
08:19:28.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.866701 12907 consensus_queue.cc:1035] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:28.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.877343 12907 consensus_queue.cc:1035] T 02c80bdaca9e44b196d252a862b98a0b P 4eccd08210d84122b76190a299fa7373 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.913878 12908 raft_consensus.cc:482] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:19:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.914142 12908 raft_consensus.cc:504] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.914742 12908 leader_election.cc:288] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4eccd08210d84122b76190a299fa7373 (127.11.219.129:33665), a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.915441 12549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ad290ad86146fc9f70a7dbdaafe667" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4eccd08210d84122b76190a299fa7373" is_pre_election: true
08:19:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.915549 12831 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ad290ad86146fc9f70a7dbdaafe667" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a78b96a30bad4c56ba1dc819a7725729" is_pre_election: true
08:19:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.915753 12831 raft_consensus.cc:2454] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 0.
08:19:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.915752 12549 raft_consensus.cc:2454] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 0.
08:19:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.916163 12615 leader_election.cc:302] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4eccd08210d84122b76190a299fa7373, 9a4f0cce1da9425a9bb0b9566d2aef09; no voters:
08:19:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.916461 12908 raft_consensus.cc:2781] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:19:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.916569 12908 raft_consensus.cc:482] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:19:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.916667 12908 raft_consensus.cc:3037] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.918581 12908 raft_consensus.cc:504] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.919135 12908 leader_election.cc:288] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 election: Requested vote from peers 4eccd08210d84122b76190a299fa7373 (127.11.219.129:33665), a78b96a30bad4c56ba1dc819a7725729 (127.11.219.131:41169)
08:19:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.919667 12549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ad290ad86146fc9f70a7dbdaafe667" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4eccd08210d84122b76190a299fa7373"
08:19:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.919731 12831 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "77ad290ad86146fc9f70a7dbdaafe667" candidate_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a78b96a30bad4c56ba1dc819a7725729"
08:19:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.919857 12549 raft_consensus.cc:3037] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.919935 12831 raft_consensus.cc:3037] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [term 0 FOLLOWER]: Advancing to term 1
08:19:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.921789 12549 raft_consensus.cc:2454] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 1.
08:19:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.921950 12831 raft_consensus.cc:2454] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9a4f0cce1da9425a9bb0b9566d2aef09 in term 1.
08:19:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.922226 12615 leader_election.cc:302] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4eccd08210d84122b76190a299fa7373, 9a4f0cce1da9425a9bb0b9566d2aef09; no voters:
08:19:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.922503 12908 raft_consensus.cc:2781] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 1 FOLLOWER]: Leader election won for term 1
08:19:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.922709 12908 raft_consensus.cc:686] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [term 1 LEADER]: Becoming Leader. State: Replica: 9a4f0cce1da9425a9bb0b9566d2aef09, State: Running, Role: LEADER
08:19:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.923010 12908 consensus_queue.cc:237] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } }
08:19:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.924652 12920 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: TxnStatusTablet state changed. Reason: New leader 9a4f0cce1da9425a9bb0b9566d2aef09. Latest consensus state: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.924973 12920 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: This TxnStatusTablet replica's current role is: LEADER
08:19:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.925580 12940 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
08:19:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.925920 12940 txn_status_manager.cc:929] Loading transaction status metadata into memory...
08:19:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.926913 12345 catalog_manager.cc:5484] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130). New cstate: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } health_report { overall_health: UNKNOWN } } }
08:19:28.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.945943 12920 consensus_queue.cc:1035] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:28.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:28.957046 12920 consensus_queue.cc:1035] T bcdb3b61c8dc43399f125e655150f8b8 P 9a4f0cce1da9425a9bb0b9566d2aef09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.003000 12831 raft_consensus.cc:1261] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729 [term 1 FOLLOWER]: Refusing update from remote peer 9a4f0cce1da9425a9bb0b9566d2aef09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:19:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.003105 12549 raft_consensus.cc:1261] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373 [term 1 FOLLOWER]: Refusing update from remote peer 9a4f0cce1da9425a9bb0b9566d2aef09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:19:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.003695 12920 consensus_queue.cc:1035] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.004004 12908 consensus_queue.cc:1035] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:19:29.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.012209 12911 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: TxnStatusTablet state changed. Reason: New leader 9a4f0cce1da9425a9bb0b9566d2aef09. Latest consensus state: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:29.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.012499 12911 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: This TxnStatusTablet replica's current role is: FOLLOWER
08:19:29.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.013108 12907 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: TxnStatusTablet state changed. Reason: New leader 9a4f0cce1da9425a9bb0b9566d2aef09. Latest consensus state: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:29.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.013406 12907 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: This TxnStatusTablet replica's current role is: FOLLOWER
08:19:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.019083 12920 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.019368 12920 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: This TxnStatusTablet replica's current role is: LEADER
08:19:29.023 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
08:19:29.026 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table 6d150404b8dc42f6857dff861e28d5e8
08:19:29.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.023782 12908 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:29.028 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 02c80bdaca9e44b196d252a862b98a0b for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
08:19:29.028 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet bcdb3b61c8dc43399f125e655150f8b8 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
08:19:29.028 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=02c80bdaca9e44b196d252a862b98a0b@[4eccd08210d84122b76190a299fa7373(127.11.219.129:33665)[L],9a4f0cce1da9425a9bb0b9566d2aef09(127.11.219.130:40423),a78b96a30bad4c56ba1dc819a7725729(127.11.219.131:41169)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=bcdb3b61c8dc43399f125e655150f8b8@[4eccd08210d84122b76190a299fa7373(127.11.219.129:33665),9a4f0cce1da9425a9bb0b9566d2aef09(127.11.219.130:40423)[L],a78b96a30bad4c56ba1dc819a7725729(127.11.219.131:41169)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
08:19:29.029 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.11.219.188:38941, [1ms] Sub RPC GetTableLocations: received response from server master-127.11.219.188:38941: OK, deferred=null) after lookup
08:19:29.029 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 4eccd08210d84122b76190a299fa7373(127.11.219.129:33665)] connecting to peer
08:19:29.030 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 4eccd08210d84122b76190a299fa7373(127.11.219.129:33665)] Successfully connected to peer
08:19:29.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.024036 12908 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P 9a4f0cce1da9425a9bb0b9566d2aef09: This TxnStatusTablet replica's current role is: LEADER
08:19:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.043676 12907 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.043980 12907 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P 4eccd08210d84122b76190a299fa7373: This TxnStatusTablet replica's current role is: FOLLOWER
08:19:29.056 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xdaadd980, L:/127.0.0.1:36140 - R:/127.11.219.129:33665] using TOKEN/null
08:19:29.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.062019 12911 tablet_replica.cc:439] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4eccd08210d84122b76190a299fa7373" member_type: VOTER last_known_addr { host: "127.11.219.129" port: 33665 } } peers { permanent_uuid: "9a4f0cce1da9425a9bb0b9566d2aef09" member_type: VOTER last_known_addr { host: "127.11.219.130" port: 40423 } } peers { permanent_uuid: "a78b96a30bad4c56ba1dc819a7725729" member_type: VOTER last_known_addr { host: "127.11.219.131" port: 41169 } } }
08:19:29.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.062305 12911 tablet_replica.cc:442] T 77ad290ad86146fc9f70a7dbdaafe667 P a78b96a30bad4c56ba1dc819a7725729: This TxnStatusTablet replica's current role is: FOLLOWER
08:19:29.114 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.219.188:38941
08:19:29.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:19:29.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.115854 12142 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task4WlLjx/build/asan/bin/kudu pid 12312
08:19:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.117182 12386 generic_service.cc:196] Checking for leaks (request via RPC)
08:19:29.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.310465 12142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4WlLjx/build/asan/bin/kudu with pid 12312
08:19:29.336 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.11.219.188:38941(127.11.219.188:38941)] handling channelInactive
08:19:29.336 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state READY due to: connection closed
08:19:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.336162 12168 connection.cc:535] server connection from 127.0.0.1:46670 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:19:29.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:19:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.367422 12735 heartbeater.cc:644] Failed to heartbeat to 127.11.219.188:38941 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:29.532 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.533 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.11.219.188:38941(127.11.219.188:38941) for tablet Kudu Master: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.549 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.551 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.569 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.589 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.590 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.590 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.590 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.609 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.611 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.611511 12958 raft_consensus.cc:482] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 24637a9d4ba148bca5d1732e4557ea61)
08:19:29.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.611871 12958 raft_consensus.cc:504] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.612805 12959 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 24637a9d4ba148bca5d1732e4557ea61)
08:19:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.612987 12959 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.613312 12958 leader_election.cc:288] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7e88e5b9f5f749b4b8aa13d9dba20db0 (127.11.219.190:44505), 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941)
08:19:29.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.613794 12193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f8de291378e94eaba67cec25238adf61" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" is_pre_election: true
08:19:29.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.614776 12193 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f8de291378e94eaba67cec25238adf61 in term 1.
08:19:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.615494 12250 leader_election.cc:302] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7e88e5b9f5f749b4b8aa13d9dba20db0, f8de291378e94eaba67cec25238adf61; no voters:
08:19:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.615689 12273 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "f8de291378e94eaba67cec25238adf61" is_pre_election: true
08:19:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.615904 12273 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7e88e5b9f5f749b4b8aa13d9dba20db0 in term 1.
08:19:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.616155 12249 leader_election.cc:334] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.616176 12958 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:19:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.616372 12958 raft_consensus.cc:482] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 24637a9d4ba148bca5d1732e4557ea61)
08:19:29.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.616469 12958 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 1 FOLLOWER]: Advancing to term 2
08:19:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.617178 12171 leader_election.cc:302] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7e88e5b9f5f749b4b8aa13d9dba20db0, f8de291378e94eaba67cec25238adf61; no voters:
08:19:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.619483 12958 raft_consensus.cc:504] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.621917 12958 leader_election.cc:288] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [CANDIDATE]: Term 2 election: Requested vote from peers 7e88e5b9f5f749b4b8aa13d9dba20db0 (127.11.219.190:44505), 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941)
08:19:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.622570 12249 leader_election.cc:334] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:29.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.622714 12959 leader_election.cc:288] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers f8de291378e94eaba67cec25238adf61 (127.11.219.189:35561), 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941)
08:19:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.622916 12959 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:19:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.623050 12959 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 24637a9d4ba148bca5d1732e4557ea61)
08:19:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.623165 12959 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 1 FOLLOWER]: Advancing to term 2
08:19:29.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.625463 12959 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:29.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.626144 12959 leader_election.cc:288] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 2 election: Requested vote from peers f8de291378e94eaba67cec25238adf61 (127.11.219.189:35561), 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941)
08:19:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.628243 12168 leader_election.cc:334] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.628682 12193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f8de291378e94eaba67cec25238adf61" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0"
08:19:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.628675 12273 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "f8de291378e94eaba67cec25238adf61"
08:19:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.628939 12193 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate f8de291378e94eaba67cec25238adf61 in current term 2: Already voted for candidate 7e88e5b9f5f749b4b8aa13d9dba20db0 in this term.
08:19:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.628943 12273 raft_consensus.cc:2379] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 7e88e5b9f5f749b4b8aa13d9dba20db0 in current term 2: Already voted for candidate f8de291378e94eaba67cec25238adf61 in this term.
08:19:29.629 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.629451 12250 leader_election.cc:302] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f8de291378e94eaba67cec25238adf61; no voters: 24637a9d4ba148bca5d1732e4557ea61, 7e88e5b9f5f749b4b8aa13d9dba20db0
08:19:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.629748 12958 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:19:29.630 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.632035 12168 leader_election.cc:334] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.632176 12168 leader_election.cc:302] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7e88e5b9f5f749b4b8aa13d9dba20db0; no voters: 24637a9d4ba148bca5d1732e4557ea61, f8de291378e94eaba67cec25238adf61
08:19:29.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.632546 12964 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
08:19:29.689 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.769 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.770 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] sending RPC to server master-127.11.219.188:38941, [4ms] received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [21ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [22ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [42ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [43ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [61ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [61ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [81ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [82ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [101ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [102ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [162ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [162ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [241ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [241ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [241ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, deferred=Deferred@654579934(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/270185886@70c2daff -> (continuation of Deferred@2054757908 after retry RPC after error@1507565852) -> (continuation of Deferred@1579061265 after retry RPC after error@955104271) -> (continuation of Deferred@173444280 after retry RPC after error@243343279) -> (continuation of Deferred@607492214 after retry RPC after error@1902990651) -> (continuation of Deferred@1340089570 after retry RPC after error@1234839254) -> (continuation of Deferred@1105519035 after retry RPC after error@1513857073), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1521674226@143258a0 -> (continuation of Deferred@2054757908 after retry RPC after error@1507565852) -> (continuation of Deferred@1579061265 after retry RPC after error@955104271) -> (continuation of Deferred@173444280 after retry RPC after error@243343279) -> (continuation of Deferred@607492214 after retry RPC after error@1902990651) -> (continuation of Deferred@1340089570 after retry RPC after error@1234839254) -> (continuation of Deferred@1105519035 after retry RPC after error@1513857073)))
08:19:29.772 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] sending RPC to server master-127.11.219.188:38941, [4ms] received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [4ms] delaying RPC due to: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [21ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [22ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [42ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [43ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [61ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [61ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [81ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [82ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [82ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [101ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [102ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [162ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [162ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [241ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [241ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [241ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, deferred=Deferred@654579934(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/270185886@70c2daff -> (continuation of Deferred@2054757908 after retry RPC after error@1507565852) -> (continuation of Deferred@1579061265 after retry RPC after error@955104271) -> (continuation of Deferred@173444280 after retry RPC after error@243343279) -> (continuation of Deferred@607492214 after retry RPC after error@1902990651) -> (continuation of Deferred@1340089570 after retry RPC after error@1234839254) -> (continuation of Deferred@1105519035 after retry RPC after error@1513857073), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1521674226@143258a0 -> (continuation of Deferred@2054757908 after retry RPC after error@1507565852) -> (continuation of Deferred@1579061265 after retry RPC after error@955104271) -> (continuation of Deferred@173444280 after retry RPC after error@243343279) -> (continuation of Deferred@607492214 after retry RPC after error@1902990651) -> (continuation of Deferred@1340089570 after retry RPC after error@1234839254) -> (continuation of Deferred@1105519035 after retry RPC after error@1513857073)))
08:19:29.789 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.790 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.790 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.809 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.812 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.829 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.832 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.849 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.850 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.869 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.871 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.871 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.909 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.910 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.969 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.970036 12964 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:19:29.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.970419 12964 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:29.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.972185 12964 leader_election.cc:288] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers f8de291378e94eaba67cec25238adf61 (127.11.219.189:35561), 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941)
08:19:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.972513 12273 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "f8de291378e94eaba67cec25238adf61" is_pre_election: true
08:19:29.972 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [2ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [2ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [23ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [23ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [42ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [44ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [62ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [63ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [83ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [83ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [123ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [123ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [182ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, deferred=Deferred@1455587413(state=PENDING, result=null, callback=(continuation of Deferred@1934869302 after retry RPC after error@1917282483) -> (continuation of Deferred@489706381 after retry RPC after error@1597159510) -> (continuation of Deferred@848665333 after retry RPC after error@2078174948) -> (continuation of Deferred@1693316649 after retry RPC after error@777458410) -> (continuation of Deferred@814446567 after retry RPC after error@950703708) -> (continuation of Deferred@863855582 after retry RPC after error@632022809), errback=(continuation of Deferred@1934869302 after retry RPC after error@1917282483) -> (continuation of Deferred@489706381 after retry RPC after error@1597159510) -> (continuation of Deferred@848665333 after retry RPC after error@2078174948) -> (continuation of Deferred@1693316649 after retry RPC after error@777458410) -> (continuation of Deferred@814446567 after retry RPC after error@950703708) -> (continuation of Deferred@863855582 after retry RPC after error@632022809)))
08:19:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.972791 12273 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7e88e5b9f5f749b4b8aa13d9dba20db0 in term 2.
08:19:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.973292 12171 leader_election.cc:302] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7e88e5b9f5f749b4b8aa13d9dba20db0, f8de291378e94eaba67cec25238adf61; no voters:
08:19:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.973611 12964 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:19:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.973721 12168 leader_election.cc:334] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:29.973 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [2ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [2ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [23ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [23ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [24ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [42ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [44ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [62ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [63ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [82ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [83ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [83ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [123ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [123ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [182ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, deferred=Deferred@1455587413(state=PENDING, result=null, callback=(continuation of Deferred@1934869302 after retry RPC after error@1917282483) -> (continuation of Deferred@489706381 after retry RPC after error@1597159510) -> (continuation of Deferred@848665333 after retry RPC after error@2078174948) -> (continuation of Deferred@1693316649 after retry RPC after error@777458410) -> (continuation of Deferred@814446567 after retry RPC after error@950703708) -> (continuation of Deferred@863855582 after retry RPC after error@632022809), errback=(continuation of Deferred@1934869302 after retry RPC after error@1917282483) -> (continuation of Deferred@489706381 after retry RPC after error@1597159510) -> (continuation of Deferred@848665333 after retry RPC after error@2078174948) -> (continuation of Deferred@1693316649 after retry RPC after error@777458410) -> (continuation of Deferred@814446567 after retry RPC after error@950703708) -> (continuation of Deferred@863855582 after retry RPC after error@632022809)))
08:19:29.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.973728 12964 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:19:29.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.973923 12964 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 2 FOLLOWER]: Advancing to term 3
08:19:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.976013 12964 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.976588 12964 leader_election.cc:288] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 3 election: Requested vote from peers f8de291378e94eaba67cec25238adf61 (127.11.219.189:35561), 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941)
08:19:29.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.977119 12273 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "f8de291378e94eaba67cec25238adf61"
08:19:29.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.977353 12273 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 2 FOLLOWER]: Advancing to term 3
08:19:29.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:29.977977 12168 leader_election.cc:334] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:29.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.979444 12273 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7e88e5b9f5f749b4b8aa13d9dba20db0 in term 3.
08:19:29.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.979873 12171 leader_election.cc:302] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 7e88e5b9f5f749b4b8aa13d9dba20db0, f8de291378e94eaba67cec25238adf61; no voters: 24637a9d4ba148bca5d1732e4557ea61
08:19:29.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.980147 12964 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 3 FOLLOWER]: Leader election won for term 3
08:19:29.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.980351 12964 raft_consensus.cc:686] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [term 3 LEADER]: Becoming Leader. State: Replica: 7e88e5b9f5f749b4b8aa13d9dba20db0, State: Running, Role: LEADER
08:19:29.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.980780 12964 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } }
08:19:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.982287 12959 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7e88e5b9f5f749b4b8aa13d9dba20db0. Latest consensus state: current_term: 3 leader_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.982486 12959 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: This master's current role is: LEADER
08:19:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.983062 12966 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
08:19:29.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.985313 12966 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8]
08:19:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.985901 12966 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=b33d8447e8894ff29b1664edcf590b7b]
08:19:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.987946 12966 tablet_loader.cc:96] loaded metadata for tablet 02c80bdaca9e44b196d252a862b98a0b (table txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8])
08:19:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.988312 12966 tablet_loader.cc:96] loaded metadata for tablet 77ad290ad86146fc9f70a7dbdaafe667 (table kudu_system.kudu_transactions [id=b33d8447e8894ff29b1664edcf590b7b])
08:19:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.988601 12966 tablet_loader.cc:96] loaded metadata for tablet bcdb3b61c8dc43399f125e655150f8b8 (table txn_manager_fallback [id=6d150404b8dc42f6857dff861e28d5e8])
08:19:29.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.988991 12966 catalog_manager.cc:1484] Initializing Kudu cluster ID...
08:19:29.989 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:29.990 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.990 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.990154 12966 catalog_manager.cc:1259] Loaded cluster ID: 6fa145b01db642519e14c73fdcd06d1b
08:19:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.990259 12966 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
08:19:29.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:29.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.991730 12966 catalog_manager.cc:1504] Loading token signing keys...
08:19:29.992 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505; will retry
08:19:29.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.992627 12966 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0: Loaded TSK: 0
08:19:29.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:29.993463 12966 catalog_manager.cc:1514] Initializing in-progress tserver states...
08:19:30.009 [DEBUG - pool-46-thread-1] (Connection.java:748) [peer master-127.11.219.188:38941(127.11.219.188:38941)] connecting to peer
08:19:30.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:30.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.188:38941(127.11.219.188:38941)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:30.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.219.188:38941: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941
08:19:30.012 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:19:30.012 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=2, TimeoutTracker(timeout=250, elapsed=23), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [2ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [3ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.11.219.189:35561,127.11.219.188:38941,127.11.219.190:44505) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.189:35561, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.188:38941, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.219.190:44505, [22ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.188:38941: Network error: Failed to connect to peer master-127.11.219.188:38941(127.11.219.188:38941): Connection refused: /127.11.219.188:38941, [22ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.189:35561: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.11.219.190:44505: OK, deferred=Deferred@1787569194(state=PENDING, result=null, callback=(continuation of Deferred@1339472372 after retry RPC after error@1829775104), errback=(continuation of Deferred@1339472372 after retry RPC after error@1829775104))) after lookup
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.056478 12184 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46860:
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:19:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:19:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.059353 12184 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:19:30.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.063810 12273 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [term 3 FOLLOWER]: Refusing update from remote peer 7e88e5b9f5f749b4b8aa13d9dba20db0: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
08:19:30.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.064271 12620 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.031s from last keepalive heartbeat (effective timeout is 1.000s)
08:19:30.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.064765 12959 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
08:19:30.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.065721 12654 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.11.219.188:38941)
08:19:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.086871 12958 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7e88e5b9f5f749b4b8aa13d9dba20db0. Latest consensus state: current_term: 3 leader_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.087186 12958 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: This master's current role is: FOLLOWER
08:19:30.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.091082 12980 tablet_replica.cc:1298] Aborted: operation has been aborted: cancelling pending write operations
08:19:30.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.105228 12958 sys_catalog.cc:455] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:30.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.105537 12958 sys_catalog.cc:458] T 00000000000000000000000000000000 P f8de291378e94eaba67cec25238adf61 [sys.catalog]: This master's current role is: FOLLOWER
08:19:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.107666 12876 heartbeater.cc:644] Failed to heartbeat to 127.11.219.188:38941 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.108894 12594 heartbeater.cc:644] Failed to heartbeat to 127.11.219.188:38941 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111)
08:19:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.108960 12168 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 -> Peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Couldn't send request to peer 24637a9d4ba148bca5d1732e4557ea61. Status: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:19:30.110 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423): transaction ID 0 is not available for further commits: state: ABORT_IN_PROGRESS user: "slave" start_timestamp: 1714551568 last_transition_timestamp: 1714551570
08:19:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.111310 12964 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7e88e5b9f5f749b4b8aa13d9dba20db0" member_type: VOTER last_known_addr { host: "127.11.219.190" port: 44505 } } peers { permanent_uuid: "f8de291378e94eaba67cec25238adf61" member_type: VOTER last_known_addr { host: "127.11.219.189" port: 35561 } } peers { permanent_uuid: "24637a9d4ba148bca5d1732e4557ea61" member_type: VOTER last_known_addr { host: "127.11.219.188" port: 38941 } } }
08:19:30.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:30.111610 12964 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 [sys.catalog]: This master's current role is: LEADER
08:19:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.304304 12168 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111) [suppressed 13 similar messages]
08:19:30.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:30.623764 12168 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 -> Peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Couldn't send request to peer 24637a9d4ba148bca5d1732e4557ea61. Status: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:19:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.071125 12877 heartbeater.cc:503] Master 127.11.219.190:44505 was elected leader, sending a full tablet report...
08:19:31.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.111445 12595 heartbeater.cc:503] Master 127.11.219.190:44505 was elected leader, sending a full tablet report...
08:19:31.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:31.132879 12168 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 -> Peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Couldn't send request to peer 24637a9d4ba148bca5d1732e4557ea61. Status: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:19:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.257062 12153 master_runner.cc:186] Error getting master registration for 127.11.219.188:38941: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111),
08:19:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.257280 12153 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:19:31.342 [DEBUG - main] (Connection.java:575) [peer master-127.11.219.188:38941(127.11.219.188:38941)] disconnecting while in state TERMINATED
08:19:31.343 [DEBUG - main] (Connection.java:575) [peer master-127.11.219.189:35561(127.11.219.189:35561)] disconnecting while in state READY
08:19:31.343 [DEBUG - main] (Connection.java:575) [peer master-127.11.219.190:44505(127.11.219.190:44505)] disconnecting while in state READY
08:19:31.343 [DEBUG - main] (Connection.java:575) [peer 4eccd08210d84122b76190a299fa7373(127.11.219.129:33665)] disconnecting while in state READY
08:19:31.344 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.219.189:35561(127.11.219.189:35561)] handling channelInactive
08:19:31.344 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:19:31.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.189:35561(127.11.219.189:35561)] cleaning up while in state READY due to: connection closed
08:19:31.344 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.219.190:44505(127.11.219.190:44505)] handling channelInactive
08:19:31.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.219.190:44505(127.11.219.190:44505)] cleaning up while in state READY due to: connection closed
08:19:31.344 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 4eccd08210d84122b76190a299fa7373(127.11.219.129:33665)] handling channelInactive
08:19:31.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 4eccd08210d84122b76190a299fa7373(127.11.219.129:33665)] cleaning up while in state READY due to: connection closed
08:19:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.345721 12142 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task4WlLjx/build/asan/bin/kudu pid 12440
08:19:31.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.353813 12589 generic_service.cc:196] Checking for leaks (request via RPC)
08:19:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.369670 12736 heartbeater.cc:503] Master 127.11.219.190:44505 was elected leader, sending a full tablet report...
08:19:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.451712 12142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4WlLjx/build/asan/bin/kudu with pid 12440
08:19:31.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.465639 12142 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task4WlLjx/build/asan/bin/kudu pid 12598
08:19:31.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:31.465606 12615 connection.cc:535] server connection from 127.11.219.129:38717 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:19:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.472026 12730 generic_service.cc:196] Checking for leaks (request via RPC)
08:19:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.587553 12142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4WlLjx/build/asan/bin/kudu with pid 12598
08:19:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:31.604463 12755 connection.cc:535] server connection from 127.11.219.130:40239 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:19:31.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.604943 12142 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task4WlLjx/build/asan/bin/kudu pid 12740
08:19:31.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:31.606776 12168 consensus_peers.cc:485] T 00000000000000000000000000000000 P 7e88e5b9f5f749b4b8aa13d9dba20db0 -> Peer 24637a9d4ba148bca5d1732e4557ea61 (127.11.219.188:38941): Couldn't send request to peer 24637a9d4ba148bca5d1732e4557ea61. Status: Network error: Client connection negotiation failed: client connection to 127.11.219.188:38941: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:19:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.611855 12871 generic_service.cc:196] Checking for leaks (request via RPC)
08:19:31.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.706845 12142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4WlLjx/build/asan/bin/kudu with pid 12740
08:19:31.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.720511 12142 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task4WlLjx/build/asan/bin/kudu pid 12153
08:19:31.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.721647 12224 generic_service.cc:196] Checking for leaks (request via RPC)
08:19:31.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.852984 12142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4WlLjx/build/asan/bin/kudu with pid 12153
08:19:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240501 08:19:31.869571 12252 connection.cc:535] server connection from 127.0.0.1:60000 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:19:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.869879 12142 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-task4WlLjx/build/asan/bin/kudu pid 12233
08:19:31.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.870950 12305 generic_service.cc:196] Checking for leaks (request via RPC)
08:19:31.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240501 08:19:31.977205 12142 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task4WlLjx/build/asan/bin/kudu with pid 12233
08:19:32.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-01T08:19:32Z chronyd exiting
08:19:32.117 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1714551568 last_transition_timestamp: 1714551570
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 9a4f0cce1da9425a9bb0b9566d2aef09 (127.11.219.130:40423): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1714551568 last_transition_timestamp: 1714551570
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more