03:11:31.349 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:11:31.350 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskEcVEWg/test-tmp
03:11:31.350 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/release/bin
03:11:31.350 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskEcVEWg/java/build/dist-test/../../../build/release/bin/kudu, test, mini_cluster, --serialization=pb]
03:11:31.351 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:11:31.419 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:11:31.420 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:11:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.421353 23251 mini_oidc.cc:150] Starting JWKS server
03:11:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.421963 23251 webserver.cc:458] Webserver started at http://127.0.0.1:43277/ using document root <none> and password file <none>
03:11:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.422034 23251 mini_oidc.cc:186] Starting OIDC Discovery server
03:11:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.422165 23251 webserver.cc:458] Webserver started at http://127.0.0.1:36299/ using document root <none> and password file <none>
03:11:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-16T03:11:31Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:11:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-16T03:11:31Z Disabled control of system clock
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.432070 23251 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/wal
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/logs
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data/info.pb
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.254:43301
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.254
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:11:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:11:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.503245 23262 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:11:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.503391 23262 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:11:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.503408 23262 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.504490 23262 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.504519 23262 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.504530 23262 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.504541 23262 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.504552 23262 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.504562 23262 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:11:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.504572 23262 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43277/jwks.json
03:11:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.505851 23262 master_runner.cc:386] Master server non-default flags:
03:11:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/wal
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.254:43301
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data/info.pb
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.254
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23262
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/logs
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jan 2025 02:43:16 UTC on a633016ff5c9
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4198
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.505995 23262 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:11:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.506155 23262 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:11:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.511114 23270 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.511570 23268 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.512789 23267 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.513818 23262 server_base.cc:1034] running on GCE node
03:11:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.513984 23262 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:11:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.514149 23262 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:11:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.515264 23262 hybrid_clock.cc:648] HybridClock initialized: now 1736997091515249 us; error 33 us; skew 500 ppm
03:11:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.516317 23262 webserver.cc:458] Webserver started at http://127.22.180.254:34239/ using document root <none> and password file <none>
03:11:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.516450 23262 fs_manager.cc:362] Metadata directory not provided
03:11:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.516486 23262 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:11:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.516547 23262 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:11:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.517256 23262 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data/instance:
03:11:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a76654fded8646f6b96e13242b2e40db"
03:11:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.517488 23262 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/wal/instance:
03:11:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a76654fded8646f6b96e13242b2e40db"
03:11:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.518407 23262 fs_manager.cc:696] Time spent creating directory manager: real 0.001s user 0.002s sys 0.000s
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.519017 23276 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.519165 23262 fs_manager.cc:730] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.519225 23262 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data,/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/wal
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a76654fded8646f6b96e13242b2e40db"
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.519268 23262 fs_report.cc:389] FS layout report
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/wal
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/wal
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data/data
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:11:31.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:11:31.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.529752 23262 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:11:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.529953 23262 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:11:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.530040 23262 kserver.cc:163] Server-wide thread pool size limit: 3276
03:11:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.533785 23262 rpc_server.cc:307] RPC server started. Bound to: 127.22.180.254:43301
03:11:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.533810 23338 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.180.254:43301 every 8 connection(s)
03:11:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.534037 23262 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/data/info.pb
03:11:31.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.534704 23339 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:31.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.535573 23251 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskEcVEWg/build/release/bin/kudu as pid 23262
03:11:31.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.535665 23251 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-0/wal/instance
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.535857 23339 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.536434 23251 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/logs
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/info.pb
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:11:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.253:44015
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.253
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:11:31.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:11:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.540879 23339 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.541186 23277 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:31.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.541566 23339 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.180.253:44015: Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.590767 23339 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } attempt: 1
03:11:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.591444 23339 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.180.253:44015: Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:31.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.606773 23343 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:11:31.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.606918 23343 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:11:31.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.606947 23343 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:11:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.608237 23343 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:11:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.608282 23343 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:11:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.608305 23343 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:11:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.608325 23343 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:11:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.608345 23343 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:11:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.608373 23343 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:11:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.608388 23343 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43277/jwks.json
03:11:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.609819 23343 master_runner.cc:386] Master server non-default flags:
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.253:44015
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/info.pb
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.253
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23343
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/logs
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jan 2025 02:43:16 UTC on a633016ff5c9
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4198
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.610035 23343 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:11:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.610249 23343 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:11:31.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.614413 23349 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.615437 23352 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.616626 23350 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.617697 23343 server_base.cc:1034] running on GCE node
03:11:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.617854 23343 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:11:31.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.618021 23343 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:11:31.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.619143 23343 hybrid_clock.cc:648] HybridClock initialized: now 1736997091619121 us; error 33 us; skew 500 ppm
03:11:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.620340 23343 webserver.cc:458] Webserver started at http://127.22.180.253:45727/ using document root <none> and password file <none>
03:11:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.620528 23343 fs_manager.cc:362] Metadata directory not provided
03:11:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.620579 23343 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:11:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.620671 23343 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:11:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.621461 23343 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/instance:
03:11:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee071289449d4044aad67f60c3c38e8d"
03:11:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.621745 23343 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal/instance:
03:11:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee071289449d4044aad67f60c3c38e8d"
03:11:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.622912 23343 fs_manager.cc:696] Time spent creating directory manager: real 0.001s user 0.002s sys 0.000s
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.623512 23358 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.623657 23343 fs_manager.cc:730] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.623736 23343 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data,/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee071289449d4044aad67f60c3c38e8d"
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.623800 23343 fs_report.cc:389] FS layout report
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:11:31.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/data
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:11:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:11:31.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.643460 23343 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:11:31.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.643752 23343 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:11:31.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.643865 23343 kserver.cc:163] Server-wide thread pool size limit: 3276
03:11:31.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.647773 23343 rpc_server.cc:307] RPC server started. Bound to: 127.22.180.253:44015
03:11:31.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.647806 23420 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.180.253:44015 every 8 connection(s)
03:11:31.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.648110 23343 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/info.pb
03:11:31.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.648841 23421 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.650064 23421 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.650776 23251 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskEcVEWg/build/release/bin/kudu as pid 23343
03:11:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.650863 23251 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal/instance
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.651175 23251 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/wal
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/logs
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data/info.pb
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.252:42075
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.252
03:11:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:11:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:11:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.654174 23421 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.657819 23421 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.658205 23362 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.658684 23421 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.180.252:42075: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:31.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.659576 23339 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } attempt: 2
03:11:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.662173 23339 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.662683 23339 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.180.252:42075: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:31.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.705874 23339 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } attempt: 1
03:11:31.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.706717 23339 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.180.252:42075: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:31.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.707868 23421 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } attempt: 1
03:11:31.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.708498 23421 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.180.252:42075: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.727309 23424 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:11:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.727463 23424 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:11:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.727494 23424 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:11:31.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.728816 23424 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:11:31.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.728857 23424 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:11:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.728878 23424 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:11:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.728897 23424 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:11:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.728914 23424 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:11:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.728929 23424 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:11:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.728945 23424 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43277/jwks.json
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.730355 23424 master_runner.cc:386] Master server non-default flags:
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/wal
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.252:42075
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data/info.pb
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.252
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23424
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/logs
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jan 2025 02:43:16 UTC on a633016ff5c9
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4198
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.730578 23424 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:11:31.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.730798 23424 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:11:31.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.735366 23434 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.736532 23431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.737747 23432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:31.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.738727 23424 server_base.cc:1034] running on GCE node
03:11:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.738905 23424 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:11:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.739074 23424 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:11:31.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.740190 23424 hybrid_clock.cc:648] HybridClock initialized: now 1736997091740173 us; error 34 us; skew 500 ppm
03:11:31.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.741196 23424 webserver.cc:458] Webserver started at http://127.22.180.252:36947/ using document root <none> and password file <none>
03:11:31.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.741334 23424 fs_manager.cc:362] Metadata directory not provided
03:11:31.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.741379 23424 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:11:31.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.741441 23424 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:11:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.742194 23424 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data/instance:
03:11:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f90ebb86973843c090fe1d594546b372"
03:11:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.742440 23424 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/wal/instance:
03:11:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f90ebb86973843c090fe1d594546b372"
03:11:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.743494 23424 fs_manager.cc:696] Time spent creating directory manager: real 0.001s user 0.000s sys 0.001s
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.743996 23440 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.744130 23424 fs_manager.cc:730] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.744207 23424 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data,/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/wal
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f90ebb86973843c090fe1d594546b372"
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.744267 23424 fs_report.cc:389] FS layout report
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/wal
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/wal
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data/data
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:11:31.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:11:31.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.753974 23339 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } attempt: 2
03:11:31.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.754613 23339 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.180.252:42075: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.756968 23424 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:11:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.757203 23424 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:11:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.757313 23424 kserver.cc:163] Server-wide thread pool size limit: 3276
03:11:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.761173 23424 rpc_server.cc:307] RPC server started. Bound to: 127.22.180.252:42075
03:11:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.761200 23502 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.180.252:42075 every 8 connection(s)
03:11:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.761493 23424 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/data/info.pb
03:11:31.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.762168 23503 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.763397 23503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.765475 23251 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskEcVEWg/build/release/bin/kudu as pid 23424
03:11:31.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.765547 23251 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-2/wal/instance
03:11:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.767421 23503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.770649 23503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
03:11:31.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.775079 23503 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: Bootstrap starting.
03:11:31.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.775909 23503 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: Neither blocks nor log segments found. Creating new log.
03:11:31.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.776101 23503 log.cc:826] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: Log is configured to *not* fsync() on all Append() calls
03:11:31.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.776635 23503 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: No bootstrap required, opened a new log
03:11:31.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.776650 23421 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } attempt: 2
03:11:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.777829 23503 raft_consensus.cc:357] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.777921 23503 raft_consensus.cc:383] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.777940 23503 raft_consensus.cc:738] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f90ebb86973843c090fe1d594546b372, State: Initialized, Role: FOLLOWER
03:11:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.778009 23503 consensus_queue.cc:260] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.778208 23512 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.778246 23503 sys_catalog.cc:564] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: configured and running, proceeding with master startup.
03:11:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.778412 23512 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: This master's current role is: FOLLOWER
03:11:31.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.780287 23421 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Bootstrap starting.
03:11:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.781049 23421 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Neither blocks nor log segments found. Creating new log.
03:11:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.781250 23421 log.cc:826] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Log is configured to *not* fsync() on all Append() calls
03:11:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.781800 23421 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: No bootstrap required, opened a new log
03:11:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.780591 23424 master_runner.cc:186] Error getting master registration for 127.22.180.254:43301: OK, instance_id { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" instance_seqno: 1736997091532874 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:11:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.782116 23424 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:11:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.782846 23523 catalog_manager.cc:1559] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:11:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.782887 23523 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:11:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.783241 23421 raft_consensus.cc:357] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.783396 23421 raft_consensus.cc:383] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.783427 23421 raft_consensus.cc:738] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ee071289449d4044aad67f60c3c38e8d, State: Initialized, Role: FOLLOWER
03:11:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.783506 23421 consensus_queue.cc:260] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.783758 23421 sys_catalog.cc:564] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: configured and running, proceeding with master startup.
03:11:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.783738 23526 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.783862 23526 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: FOLLOWER
03:11:31.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.785708 23537 catalog_manager.cc:1559] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:11:31.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.785763 23537 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:11:31.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.785585 23343 master_runner.cc:186] Error getting master registration for 127.22.180.254:43301: OK, instance_id { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" instance_seqno: 1736997091532874 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:11:31.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.786996 23343 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:11:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.858990 23526 raft_consensus.cc:491] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:11:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.859099 23526 raft_consensus.cc:513] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.859515 23526 leader_election.cc:290] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075)
03:11:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.859730 23467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ee071289449d4044aad67f60c3c38e8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f90ebb86973843c090fe1d594546b372" is_pre_election: true
03:11:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.859875 23467 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ee071289449d4044aad67f60c3c38e8d in term 0.
03:11:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.860098 23362 leader_election.cc:304] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ee071289449d4044aad67f60c3c38e8d, f90ebb86973843c090fe1d594546b372; no voters:
03:11:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.860226 23526 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Leader pre-election won for term 1
03:11:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.860356 23526 raft_consensus.cc:491] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:11:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.860428 23526 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 0 FOLLOWER]: Advancing to term 1
03:11:31.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.861224 23526 raft_consensus.cc:513] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.861514 23526 leader_election.cc:290] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 1 election: Requested vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075)
03:11:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.861634 23467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ee071289449d4044aad67f60c3c38e8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f90ebb86973843c090fe1d594546b372"
03:11:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.861698 23467 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 0 FOLLOWER]: Advancing to term 1
03:11:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.861748 23339 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } attempt: 3
03:11:31.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.862509 23467 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ee071289449d4044aad67f60c3c38e8d in term 1.
03:11:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.862692 23362 leader_election.cc:304] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ee071289449d4044aad67f60c3c38e8d, f90ebb86973843c090fe1d594546b372; no voters:
03:11:31.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.863608 23526 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 1 FOLLOWER]: Leader election won for term 1
03:11:31.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.864650 23526 raft_consensus.cc:695] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 1 LEADER]: Becoming Leader. State: Replica: ee071289449d4044aad67f60c3c38e8d, State: Running, Role: LEADER
03:11:31.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.864935 23526 consensus_queue.cc:237] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.866109 23339 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: Bootstrap starting.
03:11:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.867162 23339 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: Neither blocks nor log segments found. Creating new log.
03:11:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.867508 23339 log.cc:826] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: Log is configured to *not* fsync() on all Append() calls
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.868275 23339 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: No bootstrap required, opened a new log
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.868868 23539 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: New leader ee071289449d4044aad67f60c3c38e8d. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.869002 23539 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: LEADER
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.869298 23542 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.869576 23542 catalog_manager.cc:1485] Initializing Kudu cluster ID...
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.870194 23339 raft_consensus.cc:357] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.870286 23339 raft_consensus.cc:383] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.870321 23339 raft_consensus.cc:738] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a76654fded8646f6b96e13242b2e40db, State: Initialized, Role: FOLLOWER
03:11:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.870749 23339 consensus_queue.cc:260] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.871213 23467 raft_consensus.cc:1270] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Refusing update from remote peer ee071289449d4044aad67f60c3c38e8d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.871338 23339 sys_catalog.cc:564] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: configured and running, proceeding with master startup.
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.871416 23303 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ee071289449d4044aad67f60c3c38e8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db" is_pre_election: true
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.871497 23303 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ee071289449d4044aad67f60c3c38e8d in term 0.
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.871531 23539 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.871403 23302 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ee071289449d4044aad67f60c3c38e8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db"
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.871670 23302 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 0 FOLLOWER]: Advancing to term 1
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.872289 23302 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ee071289449d4044aad67f60c3c38e8d in term 1.
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.872570 23512 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ee071289449d4044aad67f60c3c38e8d. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.872640 23512 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: This master's current role is: FOLLOWER
03:11:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.873574 23302 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Refusing update from remote peer ee071289449d4044aad67f60c3c38e8d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:11:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.873791 23539 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.874308 23557 catalog_manager.cc:1559] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:11:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:31.874440 23557 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:11:31.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.875531 23539 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.875607 23539 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: LEADER
03:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.875918 23543 mvcc.cc:204] Tried to move back new op lower bound from 7114740088302628864 to 7114740088280469504. Current Snapshot: MvccSnapshot[applied={T|T < 7114740088302628864}]
03:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.876080 23544 sys_catalog.cc:455] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.876216 23544 sys_catalog.cc:458] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: This master's current role is: FOLLOWER
03:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.876258 23560 mvcc.cc:204] Tried to move back new op lower bound from 7114740088302628864 to 7114740088280469504. Current Snapshot: MvccSnapshot[applied={T|T < 7114740088302628864}]
03:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.876577 23544 sys_catalog.cc:455] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: SysCatalogTable state changed. Reason: New leader ee071289449d4044aad67f60c3c38e8d. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.876694 23526 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.876750 23526 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: LEADER
03:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.876986 23544 sys_catalog.cc:458] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: This master's current role is: FOLLOWER
03:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.877004 23512 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.877096 23512 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: This master's current role is: FOLLOWER
03:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.877202 23544 sys_catalog.cc:455] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.877347 23544 sys_catalog.cc:458] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: This master's current role is: FOLLOWER
03:11:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.877593 23545 mvcc.cc:204] Tried to move back new op lower bound from 7114740088302628864 to 7114740088280469504. Current Snapshot: MvccSnapshot[applied={T|T < 7114740088302628864}]
03:11:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.878489 23542 catalog_manager.cc:1348] Generated new cluster ID: d9a94d9561654b328b257f3f77c11668
03:11:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.878540 23542 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
03:11:31.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.900110 23542 catalog_manager.cc:1371] Generated new certificate authority record
03:11:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.901072 23542 catalog_manager.cc:1505] Loading token signing keys...
03:11:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.907301 23542 catalog_manager.cc:5899] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Generated new TSK 0
03:11:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.907675 23542 catalog_manager.cc:1515] Initializing in-progress tserver states...
03:11:31.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:31.916668 23251 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/wal
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/logs
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data/info.pb
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.193:0
03:11:31.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.180.193
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.193
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:11:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:11:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.023978 23567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:11:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.024163 23567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:11:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.024192 23567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:11:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.025988 23567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:11:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.026046 23567 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:11:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.026063 23567 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:11:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.026077 23567 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:11:32.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.026098 23567 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.180.193
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.028250 23567 tablet_server_runner.cc:78] Tablet server non-default flags:
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/wal
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.193:0
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data/info.pb
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.193
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23567
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.180.193
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/logs
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jan 2025 02:43:16 UTC on a633016ff5c9
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4198
03:11:32.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.028470 23567 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:11:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.028708 23567 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:11:32.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.033475 23573 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.034744 23575 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.035703 23572 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.036564 23567 server_base.cc:1034] running on GCE node
03:11:32.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.036727 23567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:11:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.036942 23567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:11:32.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.038086 23567 hybrid_clock.cc:648] HybridClock initialized: now 1736997092038065 us; error 36 us; skew 500 ppm
03:11:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.039498 23567 webserver.cc:458] Webserver started at http://127.22.180.193:44565/ using document root <none> and password file <none>
03:11:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.039712 23567 fs_manager.cc:362] Metadata directory not provided
03:11:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.039774 23567 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:11:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.039865 23567 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:11:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.040913 23567 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data/instance:
03:11:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9582249a8d9942f88cde78cdd6f7a9c2"
03:11:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.041287 23567 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/wal/instance:
03:11:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9582249a8d9942f88cde78cdd6f7a9c2"
03:11:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.042604 23567 fs_manager.cc:696] Time spent creating directory manager: real 0.001s user 0.000s sys 0.003s
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.043265 23581 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.043432 23567 fs_manager.cc:730] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.043496 23567 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data,/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/wal
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9582249a8d9942f88cde78cdd6f7a9c2"
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.043532 23567 fs_report.cc:389] FS layout report
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/wal
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/wal
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data/data
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:11:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:11:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.064167 23567 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:11:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.064337 23567 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:11:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.064402 23567 kserver.cc:163] Server-wide thread pool size limit: 3276
03:11:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.064793 23567 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:11:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.064836 23567 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:11:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.064865 23567 ts_tablet_manager.cc:610] Registered 0 tablets
03:11:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.064885 23567 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:11:32.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.071480 23567 rpc_server.cc:307] RPC server started. Bound to: 127.22.180.193:33751
03:11:32.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.071617 23702 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.180.193:33751 every 8 connection(s)
03:11:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.071990 23567 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/data/info.pb
03:11:32.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.074327 23251 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskEcVEWg/build/release/bin/kudu as pid 23567
03:11:32.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.074426 23251 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-0/wal/instance
03:11:32.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.075177 23705 heartbeater.cc:346] Connected to a master server at 127.22.180.254:43301
03:11:32.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.075254 23705 heartbeater.cc:463] Registering TS with master...
03:11:32.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.075431 23705 heartbeater.cc:509] Master 127.22.180.254:43301 requested a full tablet report, sending...
03:11:32.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.076092 23293 ts_manager.cc:194] Registered new tserver with Master: 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751)
03:11:32.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.076649 23251 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/wal
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/logs
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data/info.pb
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.194:0
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.180.194
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.194
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:11:32.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:11:32.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.078351 23704 heartbeater.cc:346] Connected to a master server at 127.22.180.252:42075
03:11:32.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.078410 23704 heartbeater.cc:463] Registering TS with master...
03:11:32.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.078533 23704 heartbeater.cc:509] Master 127.22.180.252:42075 requested a full tablet report, sending...
03:11:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.078883 23457 ts_manager.cc:194] Registered new tserver with Master: 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751)
03:11:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.079016 23706 heartbeater.cc:346] Connected to a master server at 127.22.180.253:44015
03:11:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.079066 23706 heartbeater.cc:463] Registering TS with master...
03:11:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.079155 23706 heartbeater.cc:509] Master 127.22.180.253:44015 requested a full tablet report, sending...
03:11:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.079708 23375 ts_manager.cc:194] Registered new tserver with Master: 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751)
03:11:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.080386 23375 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.180.193:45193
03:11:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.148345 23709 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:11:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.148522 23709 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:11:32.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.148552 23709 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:11:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.149858 23709 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:11:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.149910 23709 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:11:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.149932 23709 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:11:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.149951 23709 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:11:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.149971 23709 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.180.194
03:11:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.151427 23709 tablet_server_runner.cc:78] Tablet server non-default flags:
03:11:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data
03:11:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/wal
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.194:0
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data/info.pb
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.194
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23709
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.180.194
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/logs
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jan 2025 02:43:16 UTC on a633016ff5c9
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4198
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.151618 23709 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:11:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.151862 23709 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:11:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.157291 23717 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.157675 23714 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.158617 23715 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.160045 23709 server_base.cc:1034] running on GCE node
03:11:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.160204 23709 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:11:32.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.160406 23709 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:11:32.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.161533 23709 hybrid_clock.cc:648] HybridClock initialized: now 1736997092161511 us; error 35 us; skew 500 ppm
03:11:32.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.162708 23709 webserver.cc:458] Webserver started at http://127.22.180.194:44621/ using document root <none> and password file <none>
03:11:32.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.162890 23709 fs_manager.cc:362] Metadata directory not provided
03:11:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.162940 23709 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:11:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.163035 23709 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:11:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.163847 23709 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data/instance:
03:11:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d5bb722f254792b663e066249a28fc"
03:11:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.164135 23709 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/wal/instance:
03:11:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d5bb722f254792b663e066249a28fc"
03:11:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.165266 23709 fs_manager.cc:696] Time spent creating directory manager: real 0.001s user 0.002s sys 0.000s
03:11:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.165822 23723 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.165992 23709 fs_manager.cc:730] Time spent opening block manager: real 0.000s user 0.001s sys 0.000s
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.166064 23709 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data,/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/wal
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9d5bb722f254792b663e066249a28fc"
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.166123 23709 fs_report.cc:389] FS layout report
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/wal
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/wal
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data/data
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:11:32.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:11:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:11:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:11:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:11:32.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:11:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.181567 23709 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:11:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.181778 23709 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:11:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.181886 23709 kserver.cc:163] Server-wide thread pool size limit: 3276
03:11:32.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.182348 23709 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:11:32.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.182394 23709 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:11:32.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.182425 23709 ts_tablet_manager.cc:610] Registered 0 tablets
03:11:32.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.182438 23709 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:11:32.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.189428 23709 rpc_server.cc:307] RPC server started. Bound to: 127.22.180.194:41423
03:11:32.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.189491 23845 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.180.194:41423 every 8 connection(s)
03:11:32.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.189754 23709 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/data/info.pb
03:11:32.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.190820 23847 heartbeater.cc:346] Connected to a master server at 127.22.180.254:43301
03:11:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.190999 23847 heartbeater.cc:463] Registering TS with master...
03:11:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.191325 23847 heartbeater.cc:509] Master 127.22.180.254:43301 requested a full tablet report, sending...
03:11:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.191725 23251 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskEcVEWg/build/release/bin/kudu as pid 23709
03:11:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.191815 23251 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-1/wal/instance
03:11:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.191787 23293 ts_manager.cc:194] Registered new tserver with Master: b9d5bb722f254792b663e066249a28fc (127.22.180.194:41423)
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.193725 23251 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/wal
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/logs
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data/info.pb
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.195:0
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.180.195
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.195
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.195251 23846 heartbeater.cc:346] Connected to a master server at 127.22.180.252:42075
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.195302 23846 heartbeater.cc:463] Registering TS with master...
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.195416 23846 heartbeater.cc:509] Master 127.22.180.252:42075 requested a full tablet report, sending...
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.195693 23457 ts_manager.cc:194] Registered new tserver with Master: b9d5bb722f254792b663e066249a28fc (127.22.180.194:41423)
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.195907 23848 heartbeater.cc:346] Connected to a master server at 127.22.180.253:44015
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.195948 23848 heartbeater.cc:463] Registering TS with master...
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.196033 23848 heartbeater.cc:509] Master 127.22.180.253:44015 requested a full tablet report, sending...
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.196229 23375 ts_manager.cc:194] Registered new tserver with Master: b9d5bb722f254792b663e066249a28fc (127.22.180.194:41423)
03:11:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.196709 23375 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.180.194:36949
03:11:32.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.265834 23851 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:11:32.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.265973 23851 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:11:32.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.265993 23851 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:11:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.267120 23851 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:11:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.267160 23851 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:11:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.267196 23851 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:11:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.267234 23851 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:11:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.267256 23851 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.180.195
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.268616 23851 tablet_server_runner.cc:78] Tablet server non-default flags:
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/wal
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.195:0
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data/info.pb
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.195
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:11:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.23851
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.180.195
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/logs
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jan 2025 02:43:16 UTC on a633016ff5c9
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4198
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.268774 23851 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:11:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.268954 23851 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:11:32.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.273365 23859 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.274613 23856 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.275537 23857 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:32.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.276842 23851 server_base.cc:1034] running on GCE node
03:11:32.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.276952 23851 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:11:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.277112 23851 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:11:32.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.278228 23851 hybrid_clock.cc:648] HybridClock initialized: now 1736997092278215 us; error 29 us; skew 500 ppm
03:11:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.279409 23851 webserver.cc:458] Webserver started at http://127.22.180.195:43315/ using document root <none> and password file <none>
03:11:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.279556 23851 fs_manager.cc:362] Metadata directory not provided
03:11:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.279594 23851 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:11:32.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.279661 23851 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:11:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.280452 23851 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data/instance:
03:11:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65927769bac74cd18c1fba6a943427d2"
03:11:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.280706 23851 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/wal/instance:
03:11:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65927769bac74cd18c1fba6a943427d2"
03:11:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.281766 23851 fs_manager.cc:696] Time spent creating directory manager: real 0.001s user 0.000s sys 0.002s
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.282284 23865 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.282433 23851 fs_manager.cc:730] Time spent opening block manager: real 0.000s user 0.000s sys 0.001s
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.282521 23851 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data,/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/wal
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65927769bac74cd18c1fba6a943427d2"
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:32 on dist-test-slave-fpvg"
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.282564 23851 fs_report.cc:389] FS layout report
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/wal
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/wal
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data/data
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:11:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:11:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.309870 23851 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:11:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.310073 23851 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:11:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.310148 23851 kserver.cc:163] Server-wide thread pool size limit: 3276
03:11:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.310599 23851 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:11:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.310650 23851 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:11:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.310681 23851 ts_tablet_manager.cc:610] Registered 0 tablets
03:11:32.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.310703 23851 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:11:32.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.316753 23851 rpc_server.cc:307] RPC server started. Bound to: 127.22.180.195:33009
03:11:32.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.317343 23851 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/data/info.pb
03:11:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.319047 23251 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskEcVEWg/build/release/bin/kudu as pid 23851
03:11:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.319130 23251 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/ts-2/wal/instance
03:11:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.318271 23987 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.180.195:33009 every 8 connection(s)
03:11:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.320676 23988 heartbeater.cc:346] Connected to a master server at 127.22.180.252:42075
03:11:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.320837 23988 heartbeater.cc:463] Registering TS with master...
03:11:32.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.321103 23988 heartbeater.cc:509] Master 127.22.180.252:42075 requested a full tablet report, sending...
03:11:32.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.321538 23457 ts_manager.cc:194] Registered new tserver with Master: 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.322510 23989 heartbeater.cc:346] Connected to a master server at 127.22.180.254:43301
03:11:32.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.322561 23989 heartbeater.cc:463] Registering TS with master...
03:11:32.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.322665 23989 heartbeater.cc:509] Master 127.22.180.254:43301 requested a full tablet report, sending...
03:11:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.322896 23293 ts_manager.cc:194] Registered new tserver with Master: 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.323714 23990 heartbeater.cc:346] Connected to a master server at 127.22.180.253:44015
03:11:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.323757 23990 heartbeater.cc:463] Registering TS with master...
03:11:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.323815 23990 heartbeater.cc:509] Master 127.22.180.253:44015 requested a full tablet report, sending...
03:11:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.324023 23374 ts_manager.cc:194] Registered new tserver with Master: 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.324368 23374 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.180.195:51199
03:11:32.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.333529 23251 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
03:11:32.334 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:11:32.334 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:11:32.334 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.180.254"
port: 43301
}
bound_http_address {
host: "127.22.180.254"
port: 34239
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.180.253"
port: 44015
}
bound_http_address {
host: "127.22.180.253"
port: 45727
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.180.252"
port: 42075
}
bound_http_address {
host: "127.22.180.252"
port: 36947
}
}
}
03:11:32.334 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:11:32.335 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.180.193"
port: 33751
}
bound_http_address {
host: "127.22.180.193"
port: 44565
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.180.194"
port: 41423
}
bound_http_address {
host: "127.22.180.194"
port: 44621
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.180.195"
port: 33009
}
bound_http_address {
host: "127.22.180.195"
port: 43315
}
}
}
03:11:32.335 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:11:32.337 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:11:32.338 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:32.339 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:32.340 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.180.253:44015(127.22.180.253:44015)] Successfully connected to peer
03:11:32.341 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.180.254:43301(127.22.180.254:43301)] connecting to peer
03:11:32.341 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.180.253
03:11:32.347 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.180.252:42075(127.22.180.252:42075)] Successfully connected to peer
03:11:32.349 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.180.252
03:11:32.362 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.180.254:43301(127.22.180.254:43301)] Successfully connected to peer
03:11:32.364 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x17bbc988, L:/127.0.0.1:41564 - R:/127.22.180.253:44015] using SASL/PLAIN
03:11:32.364 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.22.180.254
03:11:32.368 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:11:32.369 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=31), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [31ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, deferred=null) after lookup
03:11:32.370 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x284b8b7c, L:/127.0.0.1:53872 - R:/127.22.180.252:42075] using SASL/PLAIN
03:11:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.370644 23374 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41564:
03:11:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
03:11:32.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
03:11:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.370918 23374 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:11:32.379 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa2fcc081, L:/127.0.0.1:52076 - R:/127.22.180.254:43301] using SASL/PLAIN
03:11:32.380 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 2b28faaa33154be781f2e773f9e2dca9
03:11:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.380448 23914 tablet_service.cc:1467] Processing CreateTablet for tablet b2bb91b8cd02472da8a13f857fec606f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:11:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.380785 23914 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2bb91b8cd02472da8a13f857fec606f. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.380450 23915 tablet_service.cc:1467] Processing CreateTablet for tablet 6149c8328b884452ae58058e2eb66915 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:11:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.381090 23915 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6149c8328b884452ae58058e2eb66915. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.383594 24000 tablet_bootstrap.cc:492] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2: Bootstrap starting.
03:11:32.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.384250 24000 tablet_bootstrap.cc:654] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2: Neither blocks nor log segments found. Creating new log.
03:11:32.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.384510 24000 log.cc:826] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2: Log is configured to *not* fsync() on all Append() calls
03:11:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.385104 24000 tablet_bootstrap.cc:492] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2: No bootstrap required, opened a new log
03:11:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.385167 24000 ts_tablet_manager.cc:1397] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
03:11:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.386888 24000 raft_consensus.cc:357] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.387017 24000 raft_consensus.cc:383] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.387044 24000 raft_consensus.cc:738] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65927769bac74cd18c1fba6a943427d2, State: Initialized, Role: FOLLOWER
03:11:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.387125 24000 consensus_queue.cc:260] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.387368 24000 ts_tablet_manager.cc:1428] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
03:11:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.387653 23990 heartbeater.cc:501] Master 127.22.180.253:44015 was elected leader, sending a full tablet report...
03:11:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.388006 24000 tablet_bootstrap.cc:492] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2: Bootstrap starting.
03:11:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.388507 24000 tablet_bootstrap.cc:654] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2: Neither blocks nor log segments found. Creating new log.
03:11:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.389112 24000 tablet_bootstrap.cc:492] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2: No bootstrap required, opened a new log
03:11:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.389163 24000 ts_tablet_manager.cc:1397] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2: Time spent bootstrapping tablet: real 0.001s user 0.001s sys 0.000s
03:11:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.389281 24000 raft_consensus.cc:357] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.389341 24000 raft_consensus.cc:383] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.389359 24000 raft_consensus.cc:738] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65927769bac74cd18c1fba6a943427d2, State: Initialized, Role: FOLLOWER
03:11:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.389400 24000 consensus_queue.cc:260] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.389488 24000 ts_tablet_manager.cc:1428] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
03:11:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.391220 23621 tablet_service.cc:1467] Processing CreateTablet for tablet 6149c8328b884452ae58058e2eb66915 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:11:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.391223 23631 tablet_service.cc:1467] Processing CreateTablet for tablet b2bb91b8cd02472da8a13f857fec606f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:11:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.391528 23631 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2bb91b8cd02472da8a13f857fec606f. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.394516 24005 tablet_bootstrap.cc:492] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2: Bootstrap starting.
03:11:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.394575 23621 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6149c8328b884452ae58058e2eb66915. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.395303 24005 tablet_bootstrap.cc:654] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2: Neither blocks nor log segments found. Creating new log.
03:11:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.395535 24005 log.cc:826] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2: Log is configured to *not* fsync() on all Append() calls
03:11:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.396351 24005 tablet_bootstrap.cc:492] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2: No bootstrap required, opened a new log
03:11:32.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.396407 24005 ts_tablet_manager.cc:1397] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
03:11:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.397644 24005 raft_consensus.cc:357] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.397749 24005 raft_consensus.cc:383] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.397778 24005 raft_consensus.cc:738] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9582249a8d9942f88cde78cdd6f7a9c2, State: Initialized, Role: FOLLOWER
03:11:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.397850 24005 consensus_queue.cc:260] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.398104 24005 ts_tablet_manager.cc:1428] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
03:11:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.398180 24005 tablet_bootstrap.cc:492] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2: Bootstrap starting.
03:11:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.398239 23706 heartbeater.cc:501] Master 127.22.180.253:44015 was elected leader, sending a full tablet report...
03:11:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.398751 24005 tablet_bootstrap.cc:654] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2: Neither blocks nor log segments found. Creating new log.
03:11:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.399394 23773 tablet_service.cc:1467] Processing CreateTablet for tablet 6149c8328b884452ae58058e2eb66915 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:11:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.399662 23773 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 6149c8328b884452ae58058e2eb66915. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.399895 23772 tablet_service.cc:1467] Processing CreateTablet for tablet b2bb91b8cd02472da8a13f857fec606f (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:11:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.399987 23772 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b2bb91b8cd02472da8a13f857fec606f. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.401078 24005 tablet_bootstrap.cc:492] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2: No bootstrap required, opened a new log
03:11:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.401122 24005 ts_tablet_manager.cc:1397] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.000s
03:11:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.401247 24005 raft_consensus.cc:357] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.401294 24005 raft_consensus.cc:383] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.401310 24005 raft_consensus.cc:738] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9582249a8d9942f88cde78cdd6f7a9c2, State: Initialized, Role: FOLLOWER
03:11:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.401343 24005 consensus_queue.cc:260] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.401407 24005 ts_tablet_manager.cc:1428] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
03:11:32.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.402982 24009 tablet_bootstrap.cc:492] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc: Bootstrap starting.
03:11:32.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.403571 24009 tablet_bootstrap.cc:654] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc: Neither blocks nor log segments found. Creating new log.
03:11:32.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.403764 24009 log.cc:826] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc: Log is configured to *not* fsync() on all Append() calls
03:11:32.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.404209 24009 tablet_bootstrap.cc:492] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc: No bootstrap required, opened a new log
03:11:32.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.404251 24009 ts_tablet_manager.cc:1397] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc: Time spent bootstrapping tablet: real 0.001s user 0.001s sys 0.000s
03:11:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.405323 24009 raft_consensus.cc:357] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.405431 24009 raft_consensus.cc:383] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.405447 24009 raft_consensus.cc:738] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9d5bb722f254792b663e066249a28fc, State: Initialized, Role: FOLLOWER
03:11:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.405525 24009 consensus_queue.cc:260] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.405726 24009 ts_tablet_manager.cc:1428] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
03:11:32.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.405795 23848 heartbeater.cc:501] Master 127.22.180.253:44015 was elected leader, sending a full tablet report...
03:11:32.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.405905 24009 tablet_bootstrap.cc:492] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc: Bootstrap starting.
03:11:32.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.406553 24009 tablet_bootstrap.cc:654] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc: Neither blocks nor log segments found. Creating new log.
03:11:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.407087 24009 tablet_bootstrap.cc:492] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc: No bootstrap required, opened a new log
03:11:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.407128 24009 ts_tablet_manager.cc:1397] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc: Time spent bootstrapping tablet: real 0.001s user 0.000s sys 0.001s
03:11:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.407302 24009 raft_consensus.cc:357] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.407359 24009 raft_consensus.cc:383] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.407382 24009 raft_consensus.cc:738] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9d5bb722f254792b663e066249a28fc, State: Initialized, Role: FOLLOWER
03:11:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.407425 24009 consensus_queue.cc:260] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.407526 24009 ts_tablet_manager.cc:1428] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
03:11:32.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.431982 24007 raft_consensus.cc:491] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:11:32.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.432049 24007 raft_consensus.cc:513] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.432272 24007 leader_election.cc:290] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b9d5bb722f254792b663e066249a28fc (127.22.180.194:41423), 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.434485 23792 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2bb91b8cd02472da8a13f857fec606f" candidate_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9d5bb722f254792b663e066249a28fc" is_pre_election: true
03:11:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.434477 23935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2bb91b8cd02472da8a13f857fec606f" candidate_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65927769bac74cd18c1fba6a943427d2" is_pre_election: true
03:11:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.434587 23935 raft_consensus.cc:2463] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9582249a8d9942f88cde78cdd6f7a9c2 in term 0.
03:11:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.434597 23792 raft_consensus.cc:2463] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9582249a8d9942f88cde78cdd6f7a9c2 in term 0.
03:11:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.434790 23584 leader_election.cc:304] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65927769bac74cd18c1fba6a943427d2, 9582249a8d9942f88cde78cdd6f7a9c2; no voters:
03:11:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.434926 24007 raft_consensus.cc:2798] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:11:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.434990 24007 raft_consensus.cc:491] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:11:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.435011 24007 raft_consensus.cc:3054] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Advancing to term 1
03:11:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.435570 24007 raft_consensus.cc:513] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.435694 24007 leader_election.cc:290] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [CANDIDATE]: Term 1 election: Requested vote from peers b9d5bb722f254792b663e066249a28fc (127.22.180.194:41423), 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.435865 23792 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2bb91b8cd02472da8a13f857fec606f" candidate_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b9d5bb722f254792b663e066249a28fc"
03:11:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.435859 23935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b2bb91b8cd02472da8a13f857fec606f" candidate_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65927769bac74cd18c1fba6a943427d2"
03:11:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.435933 23792 raft_consensus.cc:3054] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Advancing to term 1
03:11:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.435933 23935 raft_consensus.cc:3054] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Advancing to term 1
03:11:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.436694 23935 raft_consensus.cc:2463] T b2bb91b8cd02472da8a13f857fec606f P 65927769bac74cd18c1fba6a943427d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9582249a8d9942f88cde78cdd6f7a9c2 in term 1.
03:11:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.436693 23792 raft_consensus.cc:2463] T b2bb91b8cd02472da8a13f857fec606f P b9d5bb722f254792b663e066249a28fc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9582249a8d9942f88cde78cdd6f7a9c2 in term 1.
03:11:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.436882 23584 leader_election.cc:304] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65927769bac74cd18c1fba6a943427d2, 9582249a8d9942f88cde78cdd6f7a9c2; no voters:
03:11:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.436968 24007 raft_consensus.cc:2798] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 1 FOLLOWER]: Leader election won for term 1
03:11:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.437096 24007 raft_consensus.cc:695] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [term 1 LEADER]: Becoming Leader. State: Replica: 9582249a8d9942f88cde78cdd6f7a9c2, State: Running, Role: LEADER
03:11:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.437162 24007 consensus_queue.cc:237] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.437826 23374 catalog_manager.cc:5526] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193). New cstate: current_term: 1 leader_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } health_report { overall_health: UNKNOWN } } }
03:11:32.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.442543 23849 tablet.cc:2367] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:11:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.533695 24011 raft_consensus.cc:491] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:11:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.533797 24011 raft_consensus.cc:513] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.534065 24011 leader_election.cc:290] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751), 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.536960 23935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6149c8328b884452ae58058e2eb66915" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65927769bac74cd18c1fba6a943427d2" is_pre_election: true
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.537058 23935 raft_consensus.cc:2463] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 0.
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.537032 23650 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6149c8328b884452ae58058e2eb66915" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" is_pre_election: true
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.537138 23650 raft_consensus.cc:2463] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 0.
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.537340 23726 leader_election.cc:304] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65927769bac74cd18c1fba6a943427d2, b9d5bb722f254792b663e066249a28fc; no voters:
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.537513 24011 raft_consensus.cc:2798] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Leader pre-election won for term 1
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.537564 24011 raft_consensus.cc:491] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:11:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.537585 24011 raft_consensus.cc:3054] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Advancing to term 1
03:11:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.538108 24011 raft_consensus.cc:513] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.538215 24011 leader_election.cc:290] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 election: Requested vote from peers 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751), 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.538386 23650 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6149c8328b884452ae58058e2eb66915" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9582249a8d9942f88cde78cdd6f7a9c2"
03:11:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.538386 23935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6149c8328b884452ae58058e2eb66915" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65927769bac74cd18c1fba6a943427d2"
03:11:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.538478 23935 raft_consensus.cc:3054] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Advancing to term 1
03:11:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.538478 23650 raft_consensus.cc:3054] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Advancing to term 1
03:11:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.539135 23935 raft_consensus.cc:2463] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 1.
03:11:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.539259 23650 raft_consensus.cc:2463] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 1.
03:11:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.539324 23726 leader_election.cc:304] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65927769bac74cd18c1fba6a943427d2, b9d5bb722f254792b663e066249a28fc; no voters:
03:11:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.539407 24011 raft_consensus.cc:2798] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 1 FOLLOWER]: Leader election won for term 1
03:11:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.539546 24011 raft_consensus.cc:695] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [term 1 LEADER]: Becoming Leader. State: Replica: b9d5bb722f254792b663e066249a28fc, State: Running, Role: LEADER
03:11:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.539621 24011 consensus_queue.cc:237] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } }
03:11:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.540571 23374 catalog_manager.cc:5526] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc reported cstate change: term changed from 0 to 1, leader changed from <none> to b9d5bb722f254792b663e066249a28fc (127.22.180.194). New cstate: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } health_report { overall_health: HEALTHY } } }
03:11:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.572743 23991 tablet.cc:2367] T 6149c8328b884452ae58058e2eb66915 P 65927769bac74cd18c1fba6a943427d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:11:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.574272 23707 tablet.cc:2367] T 6149c8328b884452ae58058e2eb66915 P 9582249a8d9942f88cde78cdd6f7a9c2: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.685178 23374 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:41580:
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:11:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.685359 23374 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.687835 23915 tablet_service.cc:1467] Processing CreateTablet for tablet 7e935810ec00437b8c5cc777050be86d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=ac74132041d1488dbfa5e7efe51e9239]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.687906 23621 tablet_service.cc:1467] Processing CreateTablet for tablet 7e935810ec00437b8c5cc777050be86d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=ac74132041d1488dbfa5e7efe51e9239]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.687904 23773 tablet_service.cc:1467] Processing CreateTablet for tablet 7e935810ec00437b8c5cc777050be86d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=ac74132041d1488dbfa5e7efe51e9239]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.688004 23915 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e935810ec00437b8c5cc777050be86d. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.688050 23621 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e935810ec00437b8c5cc777050be86d. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.688050 23773 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7e935810ec00437b8c5cc777050be86d. 1 dirs total, 0 dirs full, 0 dirs failed
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.689107 24000 tablet_bootstrap.cc:492] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: Bootstrap starting.
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.689348 24005 tablet_bootstrap.cc:492] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: Bootstrap starting.
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.689575 24000 tablet_bootstrap.cc:654] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: Neither blocks nor log segments found. Creating new log.
03:11:32.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.689674 24005 tablet_bootstrap.cc:654] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: Neither blocks nor log segments found. Creating new log.
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.689941 24009 tablet_bootstrap.cc:492] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: Bootstrap starting.
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690207 24000 tablet_bootstrap.cc:492] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: No bootstrap required, opened a new log
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690250 24000 ts_tablet_manager.cc:1397] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: Time spent bootstrapping tablet: real 0.001s user 0.001s sys 0.000s
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690340 24009 tablet_bootstrap.cc:654] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: Neither blocks nor log segments found. Creating new log.
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690367 24000 raft_consensus.cc:357] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690408 24000 raft_consensus.cc:383] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690421 24000 raft_consensus.cc:738] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65927769bac74cd18c1fba6a943427d2, State: Initialized, Role: FOLLOWER
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690490 24000 consensus_queue.cc:260] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690623 24000 ts_tablet_manager.cc:1428] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690663 24002 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.690747 24002 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: This TxnStatusTablet replica's current role is: FOLLOWER
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691030 24005 tablet_bootstrap.cc:492] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: No bootstrap required, opened a new log
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691069 24005 ts_tablet_manager.cc:1397] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: Time spent bootstrapping tablet: real 0.002s user 0.001s sys 0.000s
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691196 24005 raft_consensus.cc:357] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691252 24005 raft_consensus.cc:383] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691272 24005 raft_consensus.cc:738] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9582249a8d9942f88cde78cdd6f7a9c2, State: Initialized, Role: FOLLOWER
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691320 24005 consensus_queue.cc:260] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691426 24005 ts_tablet_manager.cc:1428] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691543 24007 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691619 24007 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: This TxnStatusTablet replica's current role is: FOLLOWER
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691927 24009 tablet_bootstrap.cc:492] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: No bootstrap required, opened a new log
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.691974 24009 ts_tablet_manager.cc:1397] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: Time spent bootstrapping tablet: real 0.002s user 0.000s sys 0.001s
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.692102 24009 raft_consensus.cc:357] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.692162 24009 raft_consensus.cc:383] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.692183 24009 raft_consensus.cc:738] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b9d5bb722f254792b663e066249a28fc, State: Initialized, Role: FOLLOWER
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.692222 24009 consensus_queue.cc:260] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.692301 24009 ts_tablet_manager.cc:1428] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: Time spent starting tablet: real 0.000s user 0.000s sys 0.000s
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.692507 24015 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.692591 24015 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: This TxnStatusTablet replica's current role is: FOLLOWER
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717077 24015 raft_consensus.cc:491] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717161 24015 raft_consensus.cc:513] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717309 24015 leader_election.cc:290] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751), 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717479 23650 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e935810ec00437b8c5cc777050be86d" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" is_pre_election: true
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717562 23650 raft_consensus.cc:2463] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 0.
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717537 23935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e935810ec00437b8c5cc777050be86d" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65927769bac74cd18c1fba6a943427d2" is_pre_election: true
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717612 23935 raft_consensus.cc:2463] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 0.
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717708 23725 leader_election.cc:304] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9582249a8d9942f88cde78cdd6f7a9c2, b9d5bb722f254792b663e066249a28fc; no voters:
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717808 24015 raft_consensus.cc:2798] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Leader pre-election won for term 1
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717839 24015 raft_consensus.cc:491] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:11:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.717857 24015 raft_consensus.cc:3054] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 0 FOLLOWER]: Advancing to term 1
03:11:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.718390 24015 raft_consensus.cc:513] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.718556 24015 leader_election.cc:290] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 election: Requested vote from peers 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751), 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.718713 23935 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e935810ec00437b8c5cc777050be86d" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65927769bac74cd18c1fba6a943427d2"
03:11:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.718742 23650 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e935810ec00437b8c5cc777050be86d" candidate_uuid: "b9d5bb722f254792b663e066249a28fc" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9582249a8d9942f88cde78cdd6f7a9c2"
03:11:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.718791 23935 raft_consensus.cc:3054] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [term 0 FOLLOWER]: Advancing to term 1
03:11:32.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.718820 23650 raft_consensus.cc:3054] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [term 0 FOLLOWER]: Advancing to term 1
03:11:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.719365 23935 raft_consensus.cc:2463] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 1.
03:11:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.719421 23650 raft_consensus.cc:2463] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b9d5bb722f254792b663e066249a28fc in term 1.
03:11:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.719538 23726 leader_election.cc:304] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65927769bac74cd18c1fba6a943427d2, b9d5bb722f254792b663e066249a28fc; no voters:
03:11:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.719651 24015 raft_consensus.cc:2798] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 1 FOLLOWER]: Leader election won for term 1
03:11:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.719722 24015 raft_consensus.cc:695] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [term 1 LEADER]: Becoming Leader. State: Replica: b9d5bb722f254792b663e066249a28fc, State: Running, Role: LEADER
03:11:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.719774 24015 consensus_queue.cc:237] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } }
03:11:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.720104 24011 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: TxnStatusTablet state changed. Reason: New leader b9d5bb722f254792b663e066249a28fc. Latest consensus state: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.720189 24011 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: This TxnStatusTablet replica's current role is: LEADER
03:11:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.720376 23374 catalog_manager.cc:5526] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc reported cstate change: term changed from 0 to 1, leader changed from <none> to b9d5bb722f254792b663e066249a28fc (127.22.180.194). New cstate: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } health_report { overall_health: UNKNOWN } } }
03:11:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.720472 24029 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
03:11:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.720572 24029 txn_status_manager.cc:930] Loading transaction status metadata into memory...
03:11:32.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.731999 23935 raft_consensus.cc:1270] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2 [term 1 FOLLOWER]: Refusing update from remote peer b9d5bb722f254792b663e066249a28fc: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:11:32.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.732051 23650 raft_consensus.cc:1270] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2 [term 1 FOLLOWER]: Refusing update from remote peer b9d5bb722f254792b663e066249a28fc: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:11:32.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.732266 24015 consensus_queue.cc:1035] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [LEADER]: Connected to new peer: Peer: permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:32.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.732393 24011 consensus_queue.cc:1035] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc [LEADER]: Connected to new peer: Peer: permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.733233 24007 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: TxnStatusTablet state changed. Reason: New leader b9d5bb722f254792b663e066249a28fc. Latest consensus state: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.733319 24007 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: This TxnStatusTablet replica's current role is: FOLLOWER
03:11:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.733372 24002 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: TxnStatusTablet state changed. Reason: New leader b9d5bb722f254792b663e066249a28fc. Latest consensus state: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.733450 24002 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: This TxnStatusTablet replica's current role is: FOLLOWER
03:11:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.734867 24015 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.734903 24011 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.734982 24011 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: This TxnStatusTablet replica's current role is: LEADER
03:11:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.735608 24002 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.735677 24002 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P 65927769bac74cd18c1fba6a943427d2: This TxnStatusTablet replica's current role is: FOLLOWER
03:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.735658 24007 tablet_replica.cc:440] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b9d5bb722f254792b663e066249a28fc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 } } peers { permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 } } peers { permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 } } }
03:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.735724 24007 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P 9582249a8d9942f88cde78cdd6f7a9c2: This TxnStatusTablet replica's current role is: FOLLOWER
03:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.736258 24032 mvcc.cc:204] Tried to move back new op lower bound from 7114740091828486144 to 7114740091780755456. Current Snapshot: MvccSnapshot[applied={T|T < 7114740091828486144}]
03:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.734948 24015 tablet_replica.cc:443] T 7e935810ec00437b8c5cc777050be86d P b9d5bb722f254792b663e066249a28fc: This TxnStatusTablet replica's current role is: LEADER
03:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.736636 24034 mvcc.cc:204] Tried to move back new op lower bound from 7114740091828486144 to 7114740091780755456. Current Snapshot: MvccSnapshot[applied={T|T < 7114740091828486144}]
03:11:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.736833 24033 mvcc.cc:204] Tried to move back new op lower bound from 7114740091828486144 to 7114740091780755456. Current Snapshot: MvccSnapshot[applied={T|T < 7114740091828486144}]
03:11:32.739 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
03:11:32.740 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 2b28faaa33154be781f2e773f9e2dca9
03:11:32.740 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.180.253:44015
03:11:32.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:11:32.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.741983 23251 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskEcVEWg/build/release/bin/kudu with pid 23343
03:11:32.748 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.180.253:44015(127.22.180.253:44015)] handling channelInactive
03:11:32.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state READY due to: connection closed
03:11:32.749 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:11:32.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.783277 23523 catalog_manager.cc:1260] Loaded cluster ID: d9a94d9561654b328b257f3f77c11668
03:11:32.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.783345 23523 catalog_manager.cc:1553] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: loading cluster ID for follower catalog manager: success
03:11:32.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.784552 23523 catalog_manager.cc:1575] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: acquiring CA information for follower catalog manager: success
03:11:32.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.784862 23523 catalog_manager.cc:1603] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:11:32.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.874948 23557 catalog_manager.cc:1260] Loaded cluster ID: d9a94d9561654b328b257f3f77c11668
03:11:32.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.875025 23557 catalog_manager.cc:1553] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: loading cluster ID for follower catalog manager: success
03:11:32.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.875773 23557 catalog_manager.cc:1575] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: acquiring CA information for follower catalog manager: success
03:11:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.875978 23557 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:11:32.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.887665 24007 consensus_queue.cc:1035] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.888733 23990 heartbeater.cc:646] Failed to heartbeat to 127.22.180.253:44015 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.889549 24007 consensus_queue.cc:1035] T b2bb91b8cd02472da8a13f857fec606f P 9582249a8d9942f88cde78cdd6f7a9c2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b9d5bb722f254792b663e066249a28fc" member_type: VOTER last_known_addr { host: "127.22.180.194" port: 41423 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.889667 23706 heartbeater.cc:646] Failed to heartbeat to 127.22.180.253:44015 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:32.890487 23848 heartbeater.cc:646] Failed to heartbeat to 127.22.180.253:44015 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:32.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.979930 24038 consensus_queue.cc:1035] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [LEADER]: Connected to new peer: Peer: permanent_uuid: "65927769bac74cd18c1fba6a943427d2" member_type: VOTER last_known_addr { host: "127.22.180.195" port: 33009 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:32.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:32.981416 24011 consensus_queue.cc:1035] T 6149c8328b884452ae58058e2eb66915 P b9d5bb722f254792b663e066249a28fc [LEADER]: Connected to new peer: Peer: permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" member_type: VOTER last_known_addr { host: "127.22.180.193" port: 33751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022127 24046 raft_consensus.cc:491] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Starting pre-election (detected failure of leader ee071289449d4044aad67f60c3c38e8d)
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022220 24047 raft_consensus.cc:491] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader ee071289449d4044aad67f60c3c38e8d)
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022246 24046 raft_consensus.cc:513] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022289 24047 raft_consensus.cc:513] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022519 24046 leader_election.cc:290] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015), f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075)
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022691 24047 leader_election.cc:290] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015)
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022706 23302 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ebb86973843c090fe1d594546b372" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db" is_pre_election: true
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022786 23302 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f90ebb86973843c090fe1d594546b372 in term 1.
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022814 23467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a76654fded8646f6b96e13242b2e40db" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "f90ebb86973843c090fe1d594546b372" is_pre_election: true
03:11:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.022881 23467 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a76654fded8646f6b96e13242b2e40db in term 1.
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023026 23442 leader_election.cc:304] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a76654fded8646f6b96e13242b2e40db, f90ebb86973843c090fe1d594546b372; no voters:
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.023025 23441 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023181 23280 leader_election.cc:304] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a76654fded8646f6b96e13242b2e40db, f90ebb86973843c090fe1d594546b372; no voters:
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023233 24047 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023288 24047 raft_consensus.cc:491] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Starting leader election (detected failure of leader ee071289449d4044aad67f60c3c38e8d)
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023311 24047 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 1 FOLLOWER]: Advancing to term 2
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023308 24046 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Leader pre-election won for term 2
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023360 24046 raft_consensus.cc:491] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Starting leader election (detected failure of leader ee071289449d4044aad67f60c3c38e8d)
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.023381 24046 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 1 FOLLOWER]: Advancing to term 2
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.023437 23441 leader_election.cc:336] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:33.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.023859 23277 leader_election.cc:336] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024070 24047 raft_consensus.cc:513] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024093 24046 raft_consensus.cc:513] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024174 24047 leader_election.cc:290] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 2 election: Requested vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015)
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024204 24046 leader_election.cc:290] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 2 election: Requested vote from peers ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015), f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075)
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024300 23302 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ebb86973843c090fe1d594546b372" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db"
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024389 23302 raft_consensus.cc:2388] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate f90ebb86973843c090fe1d594546b372 in current term 2: Already voted for candidate a76654fded8646f6b96e13242b2e40db in this term.
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024470 23467 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a76654fded8646f6b96e13242b2e40db" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "f90ebb86973843c090fe1d594546b372"
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024535 23467 raft_consensus.cc:2388] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate a76654fded8646f6b96e13242b2e40db in current term 2: Already voted for candidate f90ebb86973843c090fe1d594546b372 in this term.
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.024577 23277 leader_election.cc:336] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024694 23280 leader_election.cc:304] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a76654fded8646f6b96e13242b2e40db; no voters: ee071289449d4044aad67f60c3c38e8d, f90ebb86973843c090fe1d594546b372
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024811 24046 raft_consensus.cc:2743] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.024824 23441 leader_election.cc:336] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024853 23441 leader_election.cc:304] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f90ebb86973843c090fe1d594546b372; no voters: a76654fded8646f6b96e13242b2e40db, ee071289449d4044aad67f60c3c38e8d
03:11:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.024924 24047 raft_consensus.cc:2743] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:11:33.259 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.261 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.261 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.22.180.253:44015(127.22.180.253:44015) for tablet Kudu Master: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.280 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:33.298 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.299 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.300 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.300 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.300 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:33.319 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.320 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.320 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:33.338 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.340 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.340 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:33.358 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.360 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:33.378 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:33.399 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.400 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.400 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.400 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.445925 24047 raft_consensus.cc:491] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446008 24047 raft_consensus.cc:513] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446192 24047 leader_election.cc:290] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015)
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446401 23302 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ebb86973843c090fe1d594546b372" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db" is_pre_election: true
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446532 23302 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f90ebb86973843c090fe1d594546b372 in term 2.
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446753 23442 leader_election.cc:304] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a76654fded8646f6b96e13242b2e40db, f90ebb86973843c090fe1d594546b372; no voters:
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446851 24047 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Leader pre-election won for term 3
03:11:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446892 24047 raft_consensus.cc:491] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:11:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.446857 23441 leader_election.cc:336] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.446936 24047 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 2 FOLLOWER]: Advancing to term 3
03:11:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.447499 24047 raft_consensus.cc:513] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.447618 24047 leader_election.cc:290] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 3 election: Requested vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015)
03:11:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.447836 23302 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f90ebb86973843c090fe1d594546b372" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db"
03:11:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.447918 23302 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 2 FOLLOWER]: Advancing to term 3
03:11:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.448078 23441 leader_election.cc:336] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111)
03:11:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.448525 23302 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f90ebb86973843c090fe1d594546b372 in term 3.
03:11:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.448694 23442 leader_election.cc:304] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a76654fded8646f6b96e13242b2e40db, f90ebb86973843c090fe1d594546b372; no voters: ee071289449d4044aad67f60c3c38e8d
03:11:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.448776 24047 raft_consensus.cc:2798] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 3 FOLLOWER]: Leader election won for term 3
03:11:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.448879 24047 raft_consensus.cc:695] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [term 3 LEADER]: Becoming Leader. State: Replica: f90ebb86973843c090fe1d594546b372, State: Running, Role: LEADER
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.448963 24047 consensus_queue.cc:237] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449155 24053 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f90ebb86973843c090fe1d594546b372. Latest consensus state: current_term: 3 leader_uuid: "f90ebb86973843c090fe1d594546b372" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449254 24053 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: This master's current role is: LEADER
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449407 24054 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449664 24054 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449733 24054 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=ac74132041d1488dbfa5e7efe51e9239]
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449852 24054 tablet_loader.cc:96] loaded metadata for tablet 6149c8328b884452ae58058e2eb66915 (table txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9])
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449890 24054 tablet_loader.cc:96] loaded metadata for tablet 7e935810ec00437b8c5cc777050be86d (table kudu_system.kudu_transactions [id=ac74132041d1488dbfa5e7efe51e9239])
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449913 24054 tablet_loader.cc:96] loaded metadata for tablet b2bb91b8cd02472da8a13f857fec606f (table txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9])
03:11:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449932 24054 catalog_manager.cc:1485] Initializing Kudu cluster ID...
03:11:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449970 24054 catalog_manager.cc:1260] Loaded cluster ID: d9a94d9561654b328b257f3f77c11668
03:11:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.449983 24054 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
03:11:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.450129 24054 catalog_manager.cc:1505] Loading token signing keys...
03:11:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.450170 24054 catalog_manager.cc:5910] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372: Loaded TSK: 0
03:11:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.450260 24054 catalog_manager.cc:1515] Initializing in-progress tserver states...
03:11:33.499 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:33.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.253:44015: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015
03:11:33.501 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:11:33.501 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [2ms] received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, deferred=Deferred@813632532(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@118e5572 -> (continuation of Deferred@1676709709 after retry RPC after error@1819050947) -> (continuation of Deferred@607862262 after retry RPC after error@1199430086) -> (continuation of Deferred@546356355 after retry RPC after error@686445001) -> (continuation of Deferred@1438846383 after retry RPC after error@2094615864) -> (continuation of Deferred@1493734020 after retry RPC after error@588706933) -> (continuation of Deferred@1853176781 after retry RPC after error@1483934646) -> (continuation of Deferred@1165115022 after retry RPC after error@1372839689), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@16bf2dad -> (continuation of Deferred@1676709709 after retry RPC after error@1819050947) -> (continuation of Deferred@607862262 after retry RPC after error@1199430086) -> (continuation of Deferred@546356355 after retry RPC after error@686445001) -> (continuation of Deferred@1438846383 after retry RPC after error@2094615864) -> (continuation of Deferred@1493734020 after retry RPC after error@588706933) -> (continuation of Deferred@1853176781 after retry RPC after error@1483934646) -> (continuation of Deferred@1165115022 after retry RPC after error@1372839689))) after lookup
03:11:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.509668 23487 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:53872 (request call id 9) took 7 ms (client timeout 7 ms). Trace:
03:11:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.509745 23487 rpcz_store.cc:269] 0116 03:11:33.502506 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:11:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:33.502543 (+ 37us) service_pool.cc:225] Handling call
03:11:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:33.509650 (+ 7107us) inbound_call.cc:173] Queueing success response
03:11:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":107,"thread_start_us":56,"threads_started":1}
03:11:33.510 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=10, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [2ms] received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [243ms] sending RPC to server master-127.22.180.252:42075, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@813632532(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@118e5572 -> (continuation of Deferred@1676709709 after retry RPC after error@1819050947) -> (continuation of Deferred@607862262 after retry RPC after error@1199430086) -> (continuation of Deferred@546356355 after retry RPC after error@686445001) -> (continuation of Deferred@1438846383 after retry RPC after error@2094615864) -> (continuation of Deferred@1493734020 after retry RPC after error@588706933) -> (continuation of Deferred@1853176781 after retry RPC after error@1483934646) -> (continuation of Deferred@1165115022 after retry RPC after error@1372839689) -> (continuation of Deferred@1456581239 after retry RPC@42170477), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@16bf2dad -> (continuation of Deferred@1676709709 after retry RPC after error@1819050947) -> (continuation of Deferred@607862262 after retry RPC after error@1199430086) -> (continuation of Deferred@546356355 after retry RPC after error@686445001) -> (continuation of Deferred@1438846383 after retry RPC after error@2094615864) -> (continuation of Deferred@1493734020 after retry RPC after error@588706933) -> (continuation of Deferred@1853176781 after retry RPC after error@1483934646) -> (continuation of Deferred@1165115022 after retry RPC after error@1372839689) -> (continuation of Deferred@1456581239 after retry RPC@42170477)))
03:11:33.511 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=10, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [2ms] received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [61ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [101ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [120ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [239ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: Network error: Failed to connect to peer master-127.22.180.253:44015(127.22.180.253:44015): Connection refused: /127.22.180.253:44015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: OK, [243ms] sending RPC to server master-127.22.180.252:42075, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@813632532(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@118e5572 -> (continuation of Deferred@1676709709 after retry RPC after error@1819050947) -> (continuation of Deferred@607862262 after retry RPC after error@1199430086) -> (continuation of Deferred@546356355 after retry RPC after error@686445001) -> (continuation of Deferred@1438846383 after retry RPC after error@2094615864) -> (continuation of Deferred@1493734020 after retry RPC after error@588706933) -> (continuation of Deferred@1853176781 after retry RPC after error@1483934646) -> (continuation of Deferred@1165115022 after retry RPC after error@1372839689) -> (continuation of Deferred@1456581239 after retry RPC@42170477), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@16bf2dad -> (continuation of Deferred@1676709709 after retry RPC after error@1819050947) -> (continuation of Deferred@607862262 after retry RPC after error@1199430086) -> (continuation of Deferred@546356355 after retry RPC after error@686445001) -> (continuation of Deferred@1438846383 after retry RPC after error@2094615864) -> (continuation of Deferred@1493734020 after retry RPC after error@588706933) -> (continuation of Deferred@1853176781 after retry RPC after error@1483934646) -> (continuation of Deferred@1165115022 after retry RPC after error@1372839689) -> (continuation of Deferred@1456581239 after retry RPC@42170477)))
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.539947 23457 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53900:
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:11:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.540216 23457 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:11:33.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.552788 23302 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 3 FOLLOWER]: Refusing update from remote peer f90ebb86973843c090fe1d594546b372: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
03:11:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.553022 24053 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
03:11:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.553407 24067 sys_catalog.cc:455] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: SysCatalogTable state changed. Reason: New leader f90ebb86973843c090fe1d594546b372. Latest consensus state: current_term: 3 leader_uuid: "f90ebb86973843c090fe1d594546b372" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.553512 24067 sys_catalog.cc:458] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: This master's current role is: FOLLOWER
03:11:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.553933 24047 sys_catalog.cc:455] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "f90ebb86973843c090fe1d594546b372" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.554026 24047 sys_catalog.cc:458] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 [sys.catalog]: This master's current role is: LEADER
03:11:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.554119 24067 sys_catalog.cc:455] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "f90ebb86973843c090fe1d594546b372" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:33.554198 24067 sys_catalog.cc:458] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: This master's current role is: FOLLOWER
03:11:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:33.554327 23441 consensus_peers.cc:487] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 -> Peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Couldn't send request to peer ee071289449d4044aad67f60c3c38e8d. Status: Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:11:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.060662 23441 consensus_peers.cc:487] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 -> Peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Couldn't send request to peer ee071289449d4044aad67f60c3c38e8d. Status: Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:11:34.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.589252 23441 consensus_peers.cc:487] T 00000000000000000000000000000000 P f90ebb86973843c090fe1d594546b372 -> Peer ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015): Couldn't send request to peer ee071289449d4044aad67f60c3c38e8d. Status: Network error: Client connection negotiation failed: client connection to 127.22.180.253:44015: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:11:34.751 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 6149c8328b884452ae58058e2eb66915 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
03:11:34.752 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet b2bb91b8cd02472da8a13f857fec606f for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
03:11:34.752 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=6149c8328b884452ae58058e2eb66915@[65927769bac74cd18c1fba6a943427d2(127.22.180.195:33009),9582249a8d9942f88cde78cdd6f7a9c2(127.22.180.193:33751),b9d5bb722f254792b663e066249a28fc(127.22.180.194:41423)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=b2bb91b8cd02472da8a13f857fec606f@[65927769bac74cd18c1fba6a943427d2(127.22.180.195:33009),9582249a8d9942f88cde78cdd6f7a9c2(127.22.180.193:33751)[L],b9d5bb722f254792b663e066249a28fc(127.22.180.194:41423)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
03:11:34.752 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=1), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.22.180.252:42075, [1ms] Sub RPC GetTableLocations: received response from server master-127.22.180.252:42075: OK, deferred=null) after lookup
03:11:34.752 [DEBUG - kudu-nio-1] (Connection.java:748) [peer b9d5bb722f254792b663e066249a28fc(127.22.180.194:41423)] connecting to peer
03:11:34.753 [DEBUG - kudu-nio-0] (Connection.java:757) [peer b9d5bb722f254792b663e066249a28fc(127.22.180.194:41423)] Successfully connected to peer
03:11:34.762 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3944ed9c, L:/127.0.0.1:56312 - R:/127.22.180.194:41423] using TOKEN/null
03:11:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.763648 23807 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.22.180.253:44015)
03:11:34.771 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.180.253:44015
03:11:34.771 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.772100 23251 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskEcVEWg/build/release/bin/kudu
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/logs
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/info.pb
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.253:44015
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.253
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45727
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:11:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:11:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.843861 24093 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:11:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.844009 24093 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:11:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.844027 24093 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:11:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.845111 24093 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:11:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.845144 24093 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:11:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.845155 24093 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:11:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.845166 24093 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:11:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.845177 24093 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:11:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.845187 24093 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:11:34.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.845198 24093 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43277/jwks.json
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.846541 24093 master_runner.cc:386] Master server non-default flags:
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.180.212:38263
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.180.254:43301,127.22.180.253:44015,127.22.180.252:42075
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.180.253:44015
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:11:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43277/jwks.json
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/info.pb
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.180.253
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45727
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --heap_profile_path=/tmp/kudu.24093
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/logs
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type RELEASE
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jan 2025 02:43:16 UTC on a633016ff5c9
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4198
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.846724 24093 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:11:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.846889 24093 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:11:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.851083 24098 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:34.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.852245 24099 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.853181 24101 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:11:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.854543 24093 server_base.cc:1034] running on GCE node
03:11:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.854724 24093 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:11:34.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.854930 24093 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:11:34.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.856051 24093 hybrid_clock.cc:648] HybridClock initialized: now 1736997094856036 us; error 25 us; skew 500 ppm
03:11:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.856966 24093 webserver.cc:458] Webserver started at http://127.22.180.253:45727/ using document root <none> and password file <none>
03:11:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.857122 24093 fs_manager.cc:362] Metadata directory not provided
03:11:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.857180 24093 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:11:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.858106 24093 fs_manager.cc:714] Time spent opening directory manager: real 0.000s user 0.001s sys 0.000s
03:11:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.858642 24107 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:11:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.858794 24093 fs_manager.cc:730] Time spent opening block manager: real 0.000s user 0.000s sys 0.000s
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.858872 24093 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data,/tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee071289449d4044aad67f60c3c38e8d"
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-16 03:11:31 on dist-test-slave-fpvg"
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.859143 24093 fs_report.cc:389] FS layout report
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/wal
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/data
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:11:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:11:34.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.887352 24093 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:11:34.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.887645 24093 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:11:34.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.887745 24093 kserver.cc:163] Server-wide thread pool size limit: 3276
03:11:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.891641 24093 rpc_server.cc:307] RPC server started. Bound to: 127.22.180.253:44015
03:11:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.891670 24169 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.180.253:44015 every 8 connection(s)
03:11:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.891996 24093 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskEcVEWg/test-tmp/mini-kudu-cluster1899230347196626633/master-1/data/info.pb
03:11:34.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.893329 24171 sys_catalog.cc:263] Verifying existing consensus state
03:11:34.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.894011 24171 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Bootstrap starting.
03:11:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.896147 23251 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskEcVEWg/build/release/bin/kudu as pid 24093
03:11:34.896 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:11:34.896 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.180.252:42075
03:11:34.896 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:11:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.896799 23251 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskEcVEWg/build/release/bin/kudu with pid 23424
03:11:34.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:11:34.903 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.180.252:42075(127.22.180.252:42075)] handling channelInactive
03:11:34.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state READY due to: connection closed
03:11:34.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.903512 24171 log.cc:826] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Log is configured to *not* fsync() on all Append() calls
03:11:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.905138 24171 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:11:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.905364 24171 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Bootstrap complete.
03:11:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.907256 24171 raft_consensus.cc:357] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.907459 24171 raft_consensus.cc:738] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ee071289449d4044aad67f60c3c38e8d, State: Initialized, Role: FOLLOWER
03:11:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.907547 24171 consensus_queue.cc:260] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.907768 24171 sys_catalog.cc:564] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: configured and running, proceeding with master startup.
03:11:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.907747 24174 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.907825 24134 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 1 FOLLOWER]: Advancing to term 3
03:11:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.907882 24174 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: FOLLOWER
03:11:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.909176 24187 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "f90ebb86973843c090fe1d594546b372" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.909253 24187 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: FOLLOWER
03:11:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.909294 24174 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f90ebb86973843c090fe1d594546b372. Latest consensus state: current_term: 3 leader_uuid: "f90ebb86973843c090fe1d594546b372" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.909338 24174 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: FOLLOWER
03:11:34.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:34.909386 24110 connection.cc:441] server connection from 127.0.0.1:42070 torn down before Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:42070 (request call id 0) could send its response
03:11:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.909894 24188 catalog_manager.cc:1260] Loaded cluster ID: d9a94d9561654b328b257f3f77c11668
03:11:34.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.909929 24188 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: loading cluster ID for follower catalog manager: success
03:11:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.911477 24188 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: acquiring CA information for follower catalog manager: success
03:11:34.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.911811 24188 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:11:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.916492 24093 master_runner.cc:186] Error getting master registration for 127.22.180.252:42075: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111),
03:11:34.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.917482 24093 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:11:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.982534 23988 heartbeater.cc:501] Master 127.22.180.252:42075 was elected leader, sending a full tablet report...
03:11:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.985237 23990 heartbeater.cc:346] Connected to a master server at 127.22.180.253:44015
03:11:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.985971 24124 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "65927769bac74cd18c1fba6a943427d2" instance_seqno: 1736997092315192) as {username='slave'} at 127.22.180.195:34983; Asking this server to re-register.
03:11:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.986223 23990 heartbeater.cc:463] Registering TS with master...
03:11:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:34.986570 24124 ts_manager.cc:194] Registered new tserver with Master: 65927769bac74cd18c1fba6a943427d2 (127.22.180.195:33009)
03:11:35.099 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:35.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.100 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.100 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.22.180.252:42075(127.22.180.252:42075) for tablet Kudu Master: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.118 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.253:44015(127.22.180.253:44015)] connecting to peer
03:11:35.119 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:35.119 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.180.253:44015(127.22.180.253:44015)] Successfully connected to peer
03:11:35.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.120 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.120 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.252:42075: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.129 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xcb6e7ede, L:/127.0.0.1:42082 - R:/127.22.180.253:44015] using TOKEN/null
03:11:35.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:35.138 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:35.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.252:42075: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.140 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.141891 24194 raft_consensus.cc:491] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 3 FOLLOWER]: Starting pre-election (detected failure of leader f90ebb86973843c090fe1d594546b372)
03:11:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.141970 24194 raft_consensus.cc:513] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.142140 24194 leader_election.cc:290] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers ee071289449d4044aad67f60c3c38e8d (127.22.180.253:44015), f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075)
03:11:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.142836 23280 leader_election.cc:336] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075): Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.144498 24134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a76654fded8646f6b96e13242b2e40db" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "ee071289449d4044aad67f60c3c38e8d" is_pre_election: true
03:11:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.144770 23277 leader_election.cc:304] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a76654fded8646f6b96e13242b2e40db; no voters: ee071289449d4044aad67f60c3c38e8d, f90ebb86973843c090fe1d594546b372
03:11:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.144865 24194 raft_consensus.cc:2743] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
03:11:35.158 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:35.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.159 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.252:42075: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:35.179 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:35.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.180 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.252:42075: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.180 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:35.198 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:35.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.252:42075: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.200 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301; will retry
03:11:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.209316 24174 raft_consensus.cc:491] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 3 FOLLOWER]: Starting pre-election (detected failure of leader f90ebb86973843c090fe1d594546b372)
03:11:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.209437 24174 raft_consensus.cc:513] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.209690 24174 leader_election.cc:290] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075)
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.209887 23302 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ee071289449d4044aad67f60c3c38e8d" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db" is_pre_election: true
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.209966 23302 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ee071289449d4044aad67f60c3c38e8d in term 3.
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.209960 24111 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.210170 24109 leader_election.cc:304] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a76654fded8646f6b96e13242b2e40db, ee071289449d4044aad67f60c3c38e8d; no voters:
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.210290 24174 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 3 FOLLOWER]: Leader pre-election won for term 4
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.210373 24174 raft_consensus.cc:491] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 3 FOLLOWER]: Starting leader election (detected failure of leader f90ebb86973843c090fe1d594546b372)
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.210399 24174 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 3 FOLLOWER]: Advancing to term 4
03:11:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.210383 24111 leader_election.cc:336] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075): Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.211006 24174 raft_consensus.cc:513] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.211107 24174 leader_election.cc:290] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 4 election: Requested vote from peers a76654fded8646f6b96e13242b2e40db (127.22.180.254:43301), f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075)
03:11:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.211275 23302 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ee071289449d4044aad67f60c3c38e8d" candidate_term: 4 candidate_status { last_received { term: 3 index: 12 } } ignore_live_leader: false dest_uuid: "a76654fded8646f6b96e13242b2e40db"
03:11:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.211352 23302 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 3 FOLLOWER]: Advancing to term 4
03:11:35.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.211580 24111 leader_election.cc:336] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075): Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111)
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.211949 23302 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ee071289449d4044aad67f60c3c38e8d in term 4.
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.212116 24109 leader_election.cc:304] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a76654fded8646f6b96e13242b2e40db, ee071289449d4044aad67f60c3c38e8d; no voters: f90ebb86973843c090fe1d594546b372
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.212193 24174 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 4 FOLLOWER]: Leader election won for term 4
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.212236 24174 raft_consensus.cc:695] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [term 4 LEADER]: Becoming Leader. State: Replica: ee071289449d4044aad67f60c3c38e8d, State: Running, Role: LEADER
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.212284 24174 consensus_queue.cc:237] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 3.12, Last appended by leader: 12, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } }
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.212502 24187 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: New leader ee071289449d4044aad67f60c3c38e8d. Latest consensus state: current_term: 4 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.212569 24187 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: LEADER
03:11:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.212770 24199 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213008 24199 catalog_manager.cc:670] Loaded metadata for table txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9]
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213065 24199 catalog_manager.cc:670] Loaded metadata for table kudu_system.kudu_transactions [id=ac74132041d1488dbfa5e7efe51e9239]
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213178 24199 tablet_loader.cc:96] loaded metadata for tablet 6149c8328b884452ae58058e2eb66915 (table txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9])
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213213 24199 tablet_loader.cc:96] loaded metadata for tablet 7e935810ec00437b8c5cc777050be86d (table kudu_system.kudu_transactions [id=ac74132041d1488dbfa5e7efe51e9239])
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213233 24199 tablet_loader.cc:96] loaded metadata for tablet b2bb91b8cd02472da8a13f857fec606f (table txn_manager_fallback_rolling [id=2b28faaa33154be781f2e773f9e2dca9])
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213250 24199 catalog_manager.cc:1485] Initializing Kudu cluster ID...
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213284 24199 catalog_manager.cc:1260] Loaded cluster ID: d9a94d9561654b328b257f3f77c11668
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213302 24199 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213426 24199 catalog_manager.cc:1505] Loading token signing keys...
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213466 24199 catalog_manager.cc:5910] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d: Loaded TSK: 0
03:11:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.213546 24199 catalog_manager.cc:1515] Initializing in-progress tserver states...
03:11:35.238 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.22.180.252:42075(127.22.180.252:42075)] connecting to peer
03:11:35.240 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:11:35.240 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] sending RPC to server master-127.22.180.252:42075, [1ms] received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [30ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [31ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [59ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [60ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [140ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, deferred=Deferred@268822114(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@141c5fa6 -> (continuation of Deferred@1128260526 after retry RPC after error@734494292) -> (continuation of Deferred@349092792 after retry RPC after error@1650014184) -> (continuation of Deferred@1581746846 after retry RPC after error@1315072732) -> (continuation of Deferred@789580755 after retry RPC after error@73244590) -> (continuation of Deferred@1714218689 after retry RPC after error@994395218), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@14a48e7a -> (continuation of Deferred@1128260526 after retry RPC after error@734494292) -> (continuation of Deferred@349092792 after retry RPC after error@1650014184) -> (continuation of Deferred@1581746846 after retry RPC after error@1315072732) -> (continuation of Deferred@789580755 after retry RPC after error@73244590) -> (continuation of Deferred@1714218689 after retry RPC after error@994395218))) after lookup
03:11:35.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.252:42075(127.22.180.252:42075)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.180.252:42075: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.263089 24124 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42096:
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:11:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:11:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.263613 24171 master.cc:440] Invalid argument: unable to initialize TxnManager: Error creating table kudu_system.kudu_transactions on the master: not enough live tablet servers to create a table with the requested replication factor 3; 1 tablet servers are alive: unable to init TxnManager, will retry in 1.000s
03:11:35.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.297881 23704 heartbeater.cc:501] Master 127.22.180.252:42075 was elected leader, sending a full tablet report...
03:11:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.300359 23706 heartbeater.cc:346] Connected to a master server at 127.22.180.253:44015
03:11:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.300669 24124 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "9582249a8d9942f88cde78cdd6f7a9c2" instance_seqno: 1736997092069709) as {username='slave'} at 127.22.180.193:40171; Asking this server to re-register.
03:11:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.300840 23706 heartbeater.cc:463] Registering TS with master...
03:11:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.300894 23706 heartbeater.cc:509] Master 127.22.180.253:44015 requested a full tablet report, sending...
03:11:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.301097 24124 ts_manager.cc:194] Registered new tserver with Master: 9582249a8d9942f88cde78cdd6f7a9c2 (127.22.180.193:33751)
03:11:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.301640 24111 consensus_peers.cc:487] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d -> Peer f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075): Couldn't send request to peer f90ebb86973843c090fe1d594546b372. Status: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:11:35.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.307797 23302 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [term 4 FOLLOWER]: Refusing update from remote peer ee071289449d4044aad67f60c3c38e8d: Log matching property violated. Preceding OpId in replica: term: 3 index: 12. Preceding OpId from leader: term: 4 index: 13. (index mismatch)
03:11:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.308020 24187 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [LEADER]: Connected to new peer: Peer: permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
03:11:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.308529 24194 sys_catalog.cc:455] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: SysCatalogTable state changed. Reason: New leader ee071289449d4044aad67f60c3c38e8d. Latest consensus state: current_term: 4 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:35.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.308610 24194 sys_catalog.cc:458] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: This master's current role is: FOLLOWER
03:11:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.308939 24174 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.309072 24174 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d [sys.catalog]: This master's current role is: LEADER
03:11:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.309244 24194 sys_catalog.cc:455] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "ee071289449d4044aad67f60c3c38e8d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a76654fded8646f6b96e13242b2e40db" member_type: VOTER last_known_addr { host: "127.22.180.254" port: 43301 } } peers { permanent_uuid: "ee071289449d4044aad67f60c3c38e8d" member_type: VOTER last_known_addr { host: "127.22.180.253" port: 44015 } } peers { permanent_uuid: "f90ebb86973843c090fe1d594546b372" member_type: VOTER last_known_addr { host: "127.22.180.252" port: 42075 } } }
03:11:35.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.309348 24194 sys_catalog.cc:458] T 00000000000000000000000000000000 P a76654fded8646f6b96e13242b2e40db [sys.catalog]: This master's current role is: FOLLOWER
03:11:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.350098 24154 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:42082 (request call id 6) took 109 ms (client timeout 109 ms). Trace:
03:11:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.350180 24154 rpcz_store.cc:269] 0116 03:11:35.240896 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:11:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:35.240948 (+ 52us) service_pool.cc:225] Handling call
03:11:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:35.350082 (+109134us) inbound_call.cc:173] Queueing success response
03:11:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":321,"init.queue_time_us":53,"thread_start_us":271,"threads_started":4}
03:11:35.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.252:42075, [1ms] received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [30ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [31ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [59ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [60ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [140ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [141ms] sending RPC to server master-127.22.180.253:44015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@268822114(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@141c5fa6 -> (continuation of Deferred@1128260526 after retry RPC after error@734494292) -> (continuation of Deferred@349092792 after retry RPC after error@1650014184) -> (continuation of Deferred@1581746846 after retry RPC after error@1315072732) -> (continuation of Deferred@789580755 after retry RPC after error@73244590) -> (continuation of Deferred@1714218689 after retry RPC after error@994395218) -> (continuation of Deferred@2087502767 after retry RPC@2113353741), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@14a48e7a -> (continuation of Deferred@1128260526 after retry RPC after error@734494292) -> (continuation of Deferred@349092792 after retry RPC after error@1650014184) -> (continuation of Deferred@1581746846 after retry RPC after error@1315072732) -> (continuation of Deferred@789580755 after retry RPC after error@73244590) -> (continuation of Deferred@1714218689 after retry RPC after error@994395218) -> (continuation of Deferred@2087502767 after retry RPC@2113353741)))
03:11:35.352 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.252:42075, [1ms] received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [21ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [30ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [31ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [40ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [41ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [59ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [60ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [60ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [80ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [81ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [100ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.22.180.253:44015,127.22.180.252:42075,127.22.180.254:43301) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.253:44015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.252:42075, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.180.254:43301, [140ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.253:44015: OK, [141ms] sending RPC to server master-127.22.180.253:44015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.252:42075: Network error: Failed to connect to peer master-127.22.180.252:42075(127.22.180.252:42075): Connection refused: /127.22.180.252:42075, [142ms] Sub RPC ConnectToMaster: received response from server master-127.22.180.254:43301: OK, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@268822114(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@141c5fa6 -> (continuation of Deferred@1128260526 after retry RPC after error@734494292) -> (continuation of Deferred@349092792 after retry RPC after error@1650014184) -> (continuation of Deferred@1581746846 after retry RPC after error@1315072732) -> (continuation of Deferred@789580755 after retry RPC after error@73244590) -> (continuation of Deferred@1714218689 after retry RPC after error@994395218) -> (continuation of Deferred@2087502767 after retry RPC@2113353741), errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@14a48e7a -> (continuation of Deferred@1128260526 after retry RPC after error@734494292) -> (continuation of Deferred@349092792 after retry RPC after error@1650014184) -> (continuation of Deferred@1581746846 after retry RPC after error@1315072732) -> (continuation of Deferred@789580755 after retry RPC after error@73244590) -> (continuation of Deferred@1714218689 after retry RPC after error@994395218) -> (continuation of Deferred@2087502767 after retry RPC@2113353741)))
03:11:35.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.397238 23846 heartbeater.cc:501] Master 127.22.180.252:42075 was elected leader, sending a full tablet report...
03:11:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.399806 23848 heartbeater.cc:346] Connected to a master server at 127.22.180.253:44015
03:11:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.400121 24124 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "b9d5bb722f254792b663e066249a28fc" instance_seqno: 1736997092187633) as {username='slave'} at 127.22.180.194:54969; Asking this server to re-register.
03:11:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.400360 23848 heartbeater.cc:463] Registering TS with master...
03:11:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.400463 23848 heartbeater.cc:509] Master 127.22.180.253:44015 requested a full tablet report, sending...
03:11:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.400728 24124 ts_manager.cc:194] Registered new tserver with Master: b9d5bb722f254792b663e066249a28fc (127.22.180.194:41423)
03:11:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.608927 24154 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:42082 (request call id 7) took 250 ms (client timeout 250 ms). Trace:
03:11:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.609010 24154 rpcz_store.cc:269] 0116 03:11:35.358825 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:11:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:35.358863 (+ 38us) service_pool.cc:225] Handling call
03:11:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:35.608912 (+250049us) inbound_call.cc:173] Queueing success response
03:11:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:11:35.609 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1812042805(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@5ce61091, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@468dd119))
03:11:35.609 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1812042805(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@5ce61091, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@468dd119))
03:11:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:35.686085 23730 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.105s from last keepalive heartbeat (effective timeout is 1.000s)
03:11:35.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.686484 23813 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.22.180.252:42075)
03:11:35.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.689538 24215 tablet_replica.cc:1307] Aborted: operation has been aborted: cancelling pending write operations
03:11:35.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.843969 24111 consensus_peers.cc:487] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d -> Peer f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075): Couldn't send request to peer f90ebb86973843c090fe1d594546b372. Status: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:11:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.869022 24154 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:42082 (request call id 8) took 250 ms (client timeout 250 ms). Trace:
03:11:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:35.869108 24154 rpcz_store.cc:269] 0116 03:11:35.618838 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:11:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:35.618881 (+ 43us) service_pool.cc:225] Handling call
03:11:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:35.869010 (+250129us) inbound_call.cc:173] Queueing success response
03:11:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:11:35.869 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@50249365(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@623e22c3, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@e30d48d))
03:11:35.870 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@50249365(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@623e22c3, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@e30d48d))
03:11:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:36.129108 24154 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:42082 (request call id 9) took 250 ms (client timeout 250 ms). Trace:
03:11:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:36.129185 24154 rpcz_store.cc:269] 0116 03:11:35.878906 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:11:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:35.878944 (+ 38us) service_pool.cc:225] Handling call
03:11:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0116 03:11:36.129077 (+250133us) inbound_call.cc:173] Queueing success response
03:11:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:11:36.129 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@639486283(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@51e0d10b, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@425bd9ce))
03:11:36.130 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=251), Traces: [0ms] sending RPC to server master-127.22.180.253:44015, [251ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@639486283(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$82/1027305234@51e0d10b, errback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1543002520@425bd9ce))
03:11:36.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:36.268653 24124 catalog_manager.cc:1909] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57850:
03:11:36.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:11:36.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:11:36.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:11:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:36.268798 24124 catalog_manager.cc:6885] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:11:36.273 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: b9d5bb722f254792b663e066249a28fc (127.22.180.194:41423): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1736997092 last_transition_timestamp: 1736997095
03:11:36.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:36.335026 24111 consensus_peers.cc:487] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d -> Peer f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075): Couldn't send request to peer f90ebb86973843c090fe1d594546b372. Status: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:11:36.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250116 03:11:36.869921 24111 consensus_peers.cc:487] T 00000000000000000000000000000000 P ee071289449d4044aad67f60c3c38e8d -> Peer f90ebb86973843c090fe1d594546b372 (127.22.180.252:42075): Couldn't send request to peer f90ebb86973843c090fe1d594546b372. Status: Network error: Client connection negotiation failed: client connection to 127.22.180.252:42075: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
03:11:36.905 [DEBUG - main] (Connection.java:575) [peer master-127.22.180.252:42075(127.22.180.252:42075)] disconnecting while in state TERMINATED
03:11:36.905 [DEBUG - main] (Connection.java:575) [peer master-127.22.180.254:43301(127.22.180.254:43301)] disconnecting while in state READY
03:11:36.905 [DEBUG - main] (Connection.java:575) [peer b9d5bb722f254792b663e066249a28fc(127.22.180.194:41423)] disconnecting while in state READY
03:11:36.905 [DEBUG - main] (Connection.java:575) [peer master-127.22.180.253:44015(127.22.180.253:44015)] disconnecting while in state READY
03:11:36.906 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:11:36.907 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.180.254:43301(127.22.180.254:43301)] handling channelInactive
03:11:36.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.254:43301(127.22.180.254:43301)] cleaning up while in state READY due to: connection closed
03:11:36.907 [DEBUG - kudu-nio-0] (Connection.java:239) [peer b9d5bb722f254792b663e066249a28fc(127.22.180.194:41423)] handling channelInactive
03:11:36.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:36.907665 23251 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskEcVEWg/build/release/bin/kudu with pid 23567
03:11:36.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer b9d5bb722f254792b663e066249a28fc(127.22.180.194:41423)] cleaning up while in state READY due to: connection closed
03:11:36.908 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.180.253:44015(127.22.180.253:44015)] handling channelInactive
03:11:36.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.180.253:44015(127.22.180.253:44015)] cleaning up while in state READY due to: connection closed
03:11:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:36.913308 23251 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskEcVEWg/build/release/bin/kudu with pid 23709
03:11:36.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:36.918252 23251 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskEcVEWg/build/release/bin/kudu with pid 23851
03:11:36.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:36.922389 23251 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskEcVEWg/build/release/bin/kudu with pid 23262
03:11:36.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250116 03:11:36.926795 23251 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskEcVEWg/build/release/bin/kudu with pid 24093
03:11:36.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-16T03:11:36Z chronyd exiting
03:11:36.974 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)