08:03:35.113 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:35.113 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskU_GLRA/test-tmp
08:03:35.114 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
08:03:35.114 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskU_GLRA/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:35.115 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:35.398 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:35.398 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.400731 31844 mini_oidc.cc:150] Starting JWKS server
08:03:35.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.402570 31844 webserver.cc:480] Webserver started at http://127.0.0.1:39881/ using document root <none> and password file <none>
08:03:35.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.402796 31844 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:35.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.403221 31844 webserver.cc:480] Webserver started at http://127.0.0.1:33767/ using document root <none> and password file <none>
08:03:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:03:35Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:03:35Z Disabled control of system clock
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.428817 31844 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/wal
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/logs
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data/info.pb
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.62:42415
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.62
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:35.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.709604 31855 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.709916 31855 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.710027 31855 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.716954 31855 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.717052 31855 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.717114 31855 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.717161 31855 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.717226 31855 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.717279 31855 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.717346 31855 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39881/jwks.json
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.726172 31855 master_runner.cc:386] Master server non-default flags:
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/wal
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.62:42415
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data/info.pb
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.62
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/logs
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:24 UTC on 24a791456cd2
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7892
08:03:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.728080 31855 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.729756 31855 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.737478 31860 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.737737 31861 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:35.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.737747 31863 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.738919 31855 server_base.cc:1047] running on GCE node
08:03:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.739643 31855 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:35.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.740936 31855 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.742146 31855 hybrid_clock.cc:648] HybridClock initialized: now 1756973015742090 us; error 38 us; skew 500 ppm
08:03:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.742417 31855 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.744411 31855 webserver.cc:480] Webserver started at http://127.31.25.62:38489/ using document root <none> and password file <none>
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.744976 31855 fs_manager.cc:362] Metadata directory not provided
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.745064 31855 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.745311 31855 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.747366 31855 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data/instance:
08:03:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95def791ea5d4693ac39455d37983cb2"
08:03:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:35 on dist-test-slave-x05k"
08:03:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.747979 31855 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/wal/instance:
08:03:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95def791ea5d4693ac39455d37983cb2"
08:03:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:35 on dist-test-slave-x05k"
08:03:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.752372 31855 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
08:03:35.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.755095 31869 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.756166 31855 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.756294 31855 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data,/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/wal
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95def791ea5d4693ac39455d37983cb2"
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:35 on dist-test-slave-x05k"
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.756414 31855 fs_report.cc:389] FS layout report
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/wal
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/wal
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data/data
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.771517 31855 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.772223 31855 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:35.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.772435 31855 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:35.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.792695 31855 rpc_server.cc:307] RPC server started. Bound to: 127.31.25.62:42415
08:03:35.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.792723 31930 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.25.62:42415 every 8 connection(s)
08:03:35.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.793941 31855 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/data/info.pb
08:03:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.797236 31931 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.801764 31844 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu as pid 31855
08:03:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.801990 31844 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-0/wal/instance
08:03:35.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.802287 31931 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.804013 31844 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/logs
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/info.pb
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.61:34765
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.61
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:35.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.816236 31931 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } has no permanent_uuid. Determining permanent_uuid...
08:03:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.817178 31870 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.818397 31931 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.61:34765: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:35.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.867764 31931 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } attempt: 1
08:03:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.870177 31931 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.61:34765: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:35.938392 31931 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } attempt: 2
08:03:35.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:35.940214 31931 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.61:34765: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:36.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.031416 31931 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } attempt: 3
08:03:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.033185 31931 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.61:34765: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.104789 31935 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.105072 31935 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.105157 31935 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.112480 31935 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.112562 31935 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.112607 31935 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.112643 31935 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.112706 31935 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.112749 31935 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:36.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.112792 31935 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39881/jwks.json
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.120995 31935 master_runner.cc:386] Master server non-default flags:
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.61:34765
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/info.pb
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.61
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/logs
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:24 UTC on 24a791456cd2
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7892
08:03:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.122336 31935 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:36.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.123915 31935 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:36.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.130874 31942 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.130916 31941 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.132280 31944 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.132277 31935 server_base.cc:1047] running on GCE node
08:03:36.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.133102 31935 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:36.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.134373 31935 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:36.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.135615 31935 hybrid_clock.cc:648] HybridClock initialized: now 1756973016135548 us; error 37 us; skew 500 ppm
08:03:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.135947 31935 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.137979 31935 webserver.cc:480] Webserver started at http://127.31.25.61:35161/ using document root <none> and password file <none>
08:03:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.138633 31935 fs_manager.cc:362] Metadata directory not provided
08:03:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.138752 31935 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.139158 31935 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.141377 31935 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/instance:
08:03:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "277fd7418b324b69bb0f9066c33e1fc2"
08:03:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:36 on dist-test-slave-x05k"
08:03:36.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.142125 31935 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal/instance:
08:03:36.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "277fd7418b324b69bb0f9066c33e1fc2"
08:03:36.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:36 on dist-test-slave-x05k"
08:03:36.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.147003 31935 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.003s
08:03:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.150014 31950 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.151192 31935 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.151360 31935 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data,/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "277fd7418b324b69bb0f9066c33e1fc2"
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:36 on dist-test-slave-x05k"
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.151531 31935 fs_report.cc:389] FS layout report
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/data
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:36.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.169227 31935 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:36.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.169979 31935 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:36.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.170259 31935 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:36.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.176406 31931 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } attempt: 4
08:03:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.178388 31931 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.61:34765: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.191860 31935 rpc_server.cc:307] RPC server started. Bound to: 127.31.25.61:34765
08:03:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.191880 32011 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.25.61:34765 every 8 connection(s)
08:03:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.193152 31935 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/info.pb
08:03:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.196449 32012 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.200238 31844 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu as pid 31935
08:03:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.200419 31844 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal/instance
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.201505 31844 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/wal
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/logs
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data/info.pb
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.60:44905
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.60
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:36.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.202242 32012 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } has no permanent_uuid. Determining permanent_uuid...
08:03:36.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.216135 32012 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } has no permanent_uuid. Determining permanent_uuid...
08:03:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.223878 32012 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } has no permanent_uuid. Determining permanent_uuid...
08:03:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.224759 31954 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.25.60:44905: connect: Connection refused (error 111)
08:03:36.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.226146 32012 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.60:44905: Network error: Client connection negotiation failed: client connection to 127.31.25.60:44905: connect: Connection refused (error 111)
08:03:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.275389 32012 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } attempt: 1
08:03:36.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.277206 32012 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.60:44905: Network error: Client connection negotiation failed: client connection to 127.31.25.60:44905: connect: Connection refused (error 111)
08:03:36.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.345458 32012 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } attempt: 2
08:03:36.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.347158 32012 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.60:44905: Network error: Client connection negotiation failed: client connection to 127.31.25.60:44905: connect: Connection refused (error 111)
08:03:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.438421 32012 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } attempt: 3
08:03:36.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.440424 32012 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.60:44905: Network error: Client connection negotiation failed: client connection to 127.31.25.60:44905: connect: Connection refused (error 111)
08:03:36.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.477625 31931 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } attempt: 5
08:03:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.483217 31931 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } has no permanent_uuid. Determining permanent_uuid...
08:03:36.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.484732 31931 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.60:44905: Network error: Client connection negotiation failed: client connection to 127.31.25.60:44905: connect: Connection refused (error 111)
08:03:36.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.494130 32015 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:36.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.494393 32015 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:36.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.494511 32015 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.502153 32015 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.502266 32015 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.502332 32015 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.502388 32015 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.502477 32015 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.502549 32015 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.502615 32015 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39881/jwks.json
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.511150 32015 master_runner.cc:386] Master server non-default flags:
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/wal
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.60:44905
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data/info.pb
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.60
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/logs
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:24 UTC on 24a791456cd2
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7892
08:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.513077 32015 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:36.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.514673 32015 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.522233 32023 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.522398 32022 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.523356 32025 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.523675 32015 server_base.cc:1047] running on GCE node
08:03:36.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.524287 32015 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:36.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.525444 32015 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:36.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.526629 32015 hybrid_clock.cc:648] HybridClock initialized: now 1756973016526579 us; error 36 us; skew 500 ppm
08:03:36.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.526914 32015 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:36.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.528789 32015 webserver.cc:480] Webserver started at http://127.31.25.60:37693/ using document root <none> and password file <none>
08:03:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.529419 32015 fs_manager.cc:362] Metadata directory not provided
08:03:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.529513 32015 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.529757 32015 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.531877 32015 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data/instance:
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701"
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:36 on dist-test-slave-x05k"
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.532542 32015 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/wal/instance:
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701"
08:03:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:36 on dist-test-slave-x05k"
08:03:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.535912 31931 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } attempt: 1
08:03:36.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.537565 31931 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.31.25.60:44905: Network error: Client connection negotiation failed: client connection to 127.31.25.60:44905: connect: Connection refused (error 111)
08:03:36.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.537560 32015 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
08:03:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.540589 32031 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.541672 32015 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.541836 32015 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data,/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/wal
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701"
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:36 on dist-test-slave-x05k"
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.541996 32015 fs_report.cc:389] FS layout report
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/wal
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/wal
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data/data
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:36.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.553458 32015 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:36.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.554132 32015 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:36.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.554376 32015 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:36.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.574187 32015 rpc_server.cc:307] RPC server started. Bound to: 127.31.25.60:44905
08:03:36.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.574237 32092 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.25.60:44905 every 8 connection(s)
08:03:36.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.575593 32015 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/data/info.pb
08:03:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.578936 31844 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu as pid 32015
08:03:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.579131 31844 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-2/wal/instance
08:03:36.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.580005 32093 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:36.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.583629 32012 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } attempt: 4
08:03:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.587649 32093 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } has no permanent_uuid. Determining permanent_uuid...
08:03:36.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.605733 31931 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } attempt: 2
08:03:36.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.607671 32012 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Bootstrap starting.
08:03:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.611528 32012 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Neither blocks nor log segments found. Creating new log.
08:03:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.613488 32012 log.cc:826] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Log is configured to *not* fsync() on all Append() calls
08:03:36.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.613935 32093 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } has no permanent_uuid. Determining permanent_uuid...
08:03:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.616426 32012 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: No bootstrap required, opened a new log
08:03:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.617043 31931 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: Bootstrap starting.
08:03:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.620726 31931 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: Neither blocks nor log segments found. Creating new log.
08:03:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.621255 32093 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } has no permanent_uuid. Determining permanent_uuid...
08:03:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.621906 31931 log.cc:826] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: Log is configured to *not* fsync() on all Append() calls
08:03:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.622581 32012 raft_consensus.cc:357] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.622977 32012 raft_consensus.cc:383] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:36.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.623091 32012 raft_consensus.cc:738] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 277fd7418b324b69bb0f9066c33e1fc2, State: Initialized, Role: FOLLOWER
08:03:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.623766 32012 consensus_queue.cc:260] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.625144 31931 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: No bootstrap required, opened a new log
08:03:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.625627 32102 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.626008 32102 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.626272 32012 sys_catalog.cc:564] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: configured and running, proceeding with master startup.
08:03:36.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.629880 32102 raft_consensus.cc:491] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:36.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.630123 32102 raft_consensus.cc:513] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.632364 32102 leader_election.cc:290] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415), 8a7b2fb9732a4d81a34ac1b7f5c3f701 (127.31.25.60:44905)
08:03:36.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.633448 32093 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: Bootstrap starting.
08:03:36.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.635732 31931 raft_consensus.cc:357] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.636018 31931 raft_consensus.cc:383] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:36.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.636240 31931 raft_consensus.cc:738] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 95def791ea5d4693ac39455d37983cb2, State: Initialized, Role: FOLLOWER
08:03:36.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.637007 32093 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: Neither blocks nor log segments found. Creating new log.
08:03:36.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.636912 31931 consensus_queue.cc:260] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.638402 32093 log.cc:826] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: Log is configured to *not* fsync() on all Append() calls
08:03:36.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.639983 31931 sys_catalog.cc:564] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: configured and running, proceeding with master startup.
08:03:36.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.640287 31896 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "277fd7418b324b69bb0f9066c33e1fc2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95def791ea5d4693ac39455d37983cb2" is_pre_election: true
08:03:36.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.640921 31896 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 277fd7418b324b69bb0f9066c33e1fc2 in term 0.
08:03:36.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.641763 31952 leader_election.cc:304] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 277fd7418b324b69bb0f9066c33e1fc2, 95def791ea5d4693ac39455d37983cb2; no voters:
08:03:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.642390 32102 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.642568 32102 raft_consensus.cc:491] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.642155 32113 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.642683 32102 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 0 FOLLOWER]: Advancing to term 1
08:03:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.642761 32113 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.646445 32102 raft_consensus.cc:513] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.648304 31896 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "277fd7418b324b69bb0f9066c33e1fc2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "95def791ea5d4693ac39455d37983cb2"
08:03:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.648514 31896 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 0 FOLLOWER]: Advancing to term 1
08:03:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.648805 32102 leader_election.cc:290] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [CANDIDATE]: Term 1 election: Requested vote from peers 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415), 8a7b2fb9732a4d81a34ac1b7f5c3f701 (127.31.25.60:44905)
08:03:36.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.652477 31896 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 277fd7418b324b69bb0f9066c33e1fc2 in term 1.
08:03:36.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.654325 31952 leader_election.cc:304] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 277fd7418b324b69bb0f9066c33e1fc2, 95def791ea5d4693ac39455d37983cb2; no voters:
08:03:36.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.655241 32102 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Leader election won for term 1
08:03:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.655951 32102 raft_consensus.cc:695] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 LEADER]: Becoming Leader. State: Replica: 277fd7418b324b69bb0f9066c33e1fc2, State: Running, Role: LEADER
08:03:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.656586 32102 consensus_queue.cc:237] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.668905 32117 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 277fd7418b324b69bb0f9066c33e1fc2. Latest consensus state: current_term: 1 leader_uuid: "277fd7418b324b69bb0f9066c33e1fc2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.669217 32117 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: LEADER
08:03:36.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.671373 32093 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: No bootstrap required, opened a new log
08:03:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.672384 31956 tablet.cc:2378] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.682000 32093 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.682322 32093 raft_consensus.cc:383] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.682412 32093 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8a7b2fb9732a4d81a34ac1b7f5c3f701, State: Initialized, Role: FOLLOWER
08:03:36.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.683251 32093 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:36.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.679229 31935 master_runner.cc:186] Error getting master registration for 127.31.25.62:42415: OK, instance_id { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" instance_seqno: 1756973015788286 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:36.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.688475 31935 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:36.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.695070 32121 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.696201 32121 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: This master's current role is: FOLLOWER
08:03:36.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.696343 32093 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: configured and running, proceeding with master startup.
08:03:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.696417 32058 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "277fd7418b324b69bb0f9066c33e1fc2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" is_pre_election: true
08:03:36.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.697049 32058 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 277fd7418b324b69bb0f9066c33e1fc2 in term 0.
08:03:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.698314 32057 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "277fd7418b324b69bb0f9066c33e1fc2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701"
08:03:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.698637 32057 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 0 FOLLOWER]: Advancing to term 1
08:03:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.703019 32057 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 277fd7418b324b69bb0f9066c33e1fc2 in term 1.
08:03:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.706553 32116 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.706696 32116 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.706827 32120 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:03:36.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.709173 32120 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:03:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.726397 31855 master_runner.cc:186] Error getting master registration for 127.31.25.60:44905: OK, instance_id { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" instance_seqno: 1756973016569001 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.734683 31855 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.736523 32134 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.736630 32134 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.739866 32057 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 1 FOLLOWER]: Refusing update from remote peer 277fd7418b324b69bb0f9066c33e1fc2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:36.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.740877 32117 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.748551 31896 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 1 FOLLOWER]: Refusing update from remote peer 277fd7418b324b69bb0f9066c33e1fc2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.750074 32117 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:36.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.769543 32121 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 277fd7418b324b69bb0f9066c33e1fc2. Latest consensus state: current_term: 1 leader_uuid: "277fd7418b324b69bb0f9066c33e1fc2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.769796 32121 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: This master's current role is: FOLLOWER
08:03:36.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.774662 31875 tablet.cc:2378] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:36.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.779425 32102 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "277fd7418b324b69bb0f9066c33e1fc2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.779692 32102 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: LEADER
08:03:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.787421 32120 catalog_manager.cc:1349] Generated new cluster ID: ee4a44ad2f694eaa87bb14a661cd1fa3
08:03:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.787511 32120 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:03:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.791831 32121 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "277fd7418b324b69bb0f9066c33e1fc2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.792037 32121 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: This master's current role is: FOLLOWER
08:03:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.800047 32113 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 277fd7418b324b69bb0f9066c33e1fc2. Latest consensus state: current_term: 1 leader_uuid: "277fd7418b324b69bb0f9066c33e1fc2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.800279 32113 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.803684 32117 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "277fd7418b324b69bb0f9066c33e1fc2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.804446 32117 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: LEADER
08:03:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.811223 32113 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "277fd7418b324b69bb0f9066c33e1fc2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.811388 32113 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:36.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.846894 32120 catalog_manager.cc:1372] Generated new certificate authority record
08:03:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.848445 32120 catalog_manager.cc:1506] Loading token signing keys...
08:03:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.853605 32152 catalog_manager.cc:1261] Loaded cluster ID: ee4a44ad2f694eaa87bb14a661cd1fa3
08:03:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.853689 32152 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: loading cluster ID for follower catalog manager: success
08:03:36.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.857268 32152 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: acquiring CA information for follower catalog manager: success
08:03:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:36.858532 32152 catalog_manager.cc:875] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
08:03:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.870932 32120 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Generated new TSK 0
08:03:36.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.872040 32120 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:03:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:36.889722 31844 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/wal
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/logs
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data/info.pb
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.1:0
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.25.1
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.1
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.323343 32154 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.323719 32154 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.323855 32154 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.337312 32154 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.337473 32154 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.337533 32154 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.337580 32154 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.337689 32154 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.25.1
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.352367 32154 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/wal
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.1:0
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data/info.pb
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.1
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.25.1
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/logs
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:24 UTC on 24a791456cd2
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7892
08:03:37.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:37.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.354347 32154 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.356585 32154 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:37.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.366552 32159 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.366499 32160 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.368141 32162 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.368676 32154 server_base.cc:1047] running on GCE node
08:03:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.369315 32154 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:37.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.370316 32154 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:37.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.371562 32154 hybrid_clock.cc:648] HybridClock initialized: now 1756973017371500 us; error 37 us; skew 500 ppm
08:03:37.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.371904 32154 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.374205 32154 webserver.cc:480] Webserver started at http://127.31.25.1:44879/ using document root <none> and password file <none>
08:03:37.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.375051 32154 fs_manager.cc:362] Metadata directory not provided
08:03:37.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.375169 32154 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:37.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.375532 32154 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.378398 32154 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data/instance:
08:03:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81065a6e8be5478eab56c94b4058873e"
08:03:37.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:37 on dist-test-slave-x05k"
08:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.379406 32154 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/wal/instance:
08:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81065a6e8be5478eab56c94b4058873e"
08:03:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:37 on dist-test-slave-x05k"
08:03:37.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.385568 32154 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
08:03:37.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.389528 32168 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.391093 32154 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.391240 32154 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data,/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/wal
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81065a6e8be5478eab56c94b4058873e"
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:37 on dist-test-slave-x05k"
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.391431 32154 fs_report.cc:389] FS layout report
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/wal
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/wal
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data/data
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.433668 32154 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.434387 32154 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.434607 32154 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.437398 32154 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.437510 32154 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.437603 32154 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.437667 32154 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.498435 32154 rpc_server.cc:307] RPC server started. Bound to: 127.31.25.1:46853
08:03:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.498497 32292 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.25.1:46853 every 8 connection(s)
08:03:37.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.499660 32154 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/data/info.pb
08:03:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.500151 31844 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu as pid 32154
08:03:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.500365 31844 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-0/wal/instance
08:03:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.501813 32294 heartbeater.cc:344] Connected to a master server at 127.31.25.62:42415
08:03:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.502166 32294 heartbeater.cc:461] Registering TS with master...
08:03:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.503391 32294 heartbeater.cc:507] Master 127.31.25.62:42415 requested a full tablet report, sending...
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.505247 31844 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/wal
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/logs
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data/info.pb
08:03:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.2:0
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.25.2
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.2
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:37.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.505977 31886 ts_manager.cc:194] Registered new tserver with Master: 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853)
08:03:37.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.512669 32293 heartbeater.cc:344] Connected to a master server at 127.31.25.60:44905
08:03:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.512926 32293 heartbeater.cc:461] Registering TS with master...
08:03:37.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.513551 32293 heartbeater.cc:507] Master 127.31.25.60:44905 requested a full tablet report, sending...
08:03:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.515228 32048 ts_manager.cc:194] Registered new tserver with Master: 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853)
08:03:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.520242 32295 heartbeater.cc:344] Connected to a master server at 127.31.25.61:34765
08:03:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.520380 32295 heartbeater.cc:461] Registering TS with master...
08:03:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.520752 32295 heartbeater.cc:507] Master 127.31.25.61:34765 requested a full tablet report, sending...
08:03:37.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.521931 31966 ts_manager.cc:194] Registered new tserver with Master: 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853)
08:03:37.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.523399 31966 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.25.1:50209
08:03:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.738462 32134 catalog_manager.cc:1261] Loaded cluster ID: ee4a44ad2f694eaa87bb14a661cd1fa3
08:03:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.738559 32134 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: loading cluster ID for follower catalog manager: success
08:03:37.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.741278 32134 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: acquiring CA information for follower catalog manager: success
08:03:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.743069 32134 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:37.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.820581 32298 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:37.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.820849 32298 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:37.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.820936 32298 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.828475 32298 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.828581 32298 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.828625 32298 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.828684 32298 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.828776 32298 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.25.2
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.838025 32298 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/wal
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.2:0
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data/info.pb
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.2
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.25.2
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/logs
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:24 UTC on 24a791456cd2
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7892
08:03:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.839586 32298 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:37.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.841219 32298 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.849509 32303 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.849538 32304 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:37.850245 32306 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.850594 32298 server_base.cc:1047] running on GCE node
08:03:37.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.851172 32298 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:37.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.851918 32298 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.853112 32298 hybrid_clock.cc:648] HybridClock initialized: now 1756973017853062 us; error 53 us; skew 500 ppm
08:03:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.853369 32298 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.855183 32298 webserver.cc:480] Webserver started at http://127.31.25.2:37577/ using document root <none> and password file <none>
08:03:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.855789 32298 fs_manager.cc:362] Metadata directory not provided
08:03:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.855898 32298 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.856227 32298 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.858415 32298 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data/instance:
08:03:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c99bc40bf5164ce5949dc5c176536629"
08:03:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:37 on dist-test-slave-x05k"
08:03:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.859246 32298 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/wal/instance:
08:03:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c99bc40bf5164ce5949dc5c176536629"
08:03:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:37 on dist-test-slave-x05k"
08:03:37.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.861502 32152 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.864486 32298 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
08:03:37.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.867416 32312 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.868530 32298 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.868646 32298 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data,/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/wal
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c99bc40bf5164ce5949dc5c176536629"
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:37 on dist-test-slave-x05k"
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.868774 32298 fs_report.cc:389] FS layout report
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/wal
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/wal
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data/data
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:37.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:37.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.883646 32298 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.884271 32298 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.884479 32298 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.886942 32298 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.887069 32298 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.887179 32298 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.887257 32298 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:37.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.944139 32298 rpc_server.cc:307] RPC server started. Bound to: 127.31.25.2:33279
08:03:37.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.944213 32433 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.25.2:33279 every 8 connection(s)
08:03:37.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.945335 32298 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/data/info.pb
08:03:37.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.947347 32435 heartbeater.cc:344] Connected to a master server at 127.31.25.62:42415
08:03:37.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.947798 32435 heartbeater.cc:461] Registering TS with master...
08:03:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.948935 32435 heartbeater.cc:507] Master 127.31.25.62:42415 requested a full tablet report, sending...
08:03:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.949406 31844 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu as pid 32298
08:03:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.949590 31844 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-1/wal/instance
08:03:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.950640 31886 ts_manager.cc:194] Registered new tserver with Master: c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279)
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.953956 31844 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/wal
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/logs
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data/info.pb
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.3:0
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.25.3
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.3
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
08:03:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.960213 32434 heartbeater.cc:344] Connected to a master server at 127.31.25.60:44905
08:03:37.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.960362 32434 heartbeater.cc:461] Registering TS with master...
08:03:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.960937 32434 heartbeater.cc:507] Master 127.31.25.60:44905 requested a full tablet report, sending...
08:03:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.962059 32048 ts_manager.cc:194] Registered new tserver with Master: c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279)
08:03:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.963303 32436 heartbeater.cc:344] Connected to a master server at 127.31.25.61:34765
08:03:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.963423 32436 heartbeater.cc:461] Registering TS with master...
08:03:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.963778 32436 heartbeater.cc:507] Master 127.31.25.61:34765 requested a full tablet report, sending...
08:03:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.964537 31966 ts_manager.cc:194] Registered new tserver with Master: c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279)
08:03:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:37.965277 31966 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.25.2:60151
08:03:38.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.251320 32439 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:38.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.251566 32439 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:38.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.251647 32439 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.259088 32439 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.259199 32439 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
08:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.259244 32439 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
08:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.259280 32439 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
08:03:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.259370 32439 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.25.3
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.268055 32439 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/wal
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.3:0
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data/info.pb
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.3
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.25.3
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/logs
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:24 UTC on 24a791456cd2
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7892
08:03:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.269450 32439 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.271124 32439 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.279222 32444 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.279177 32445 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.280225 32447 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.280365 32439 server_base.cc:1047] running on GCE node
08:03:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.280876 32439 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.281607 32439 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.282848 32439 hybrid_clock.cc:648] HybridClock initialized: now 1756973018282749 us; error 30 us; skew 500 ppm
08:03:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.283080 32439 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.284857 32439 webserver.cc:480] Webserver started at http://127.31.25.3:41567/ using document root <none> and password file <none>
08:03:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.285429 32439 fs_manager.cc:362] Metadata directory not provided
08:03:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.285517 32439 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.285801 32439 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:03:38.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.287887 32439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data/instance:
08:03:38.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6bb62b6d2754b549b8a04b14f7b06d1"
08:03:38.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:38 on dist-test-slave-x05k"
08:03:38.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.288506 32439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/wal/instance:
08:03:38.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6bb62b6d2754b549b8a04b14f7b06d1"
08:03:38.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:38 on dist-test-slave-x05k"
08:03:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.292902 32439 fs_manager.cc:696] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
08:03:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.295564 32453 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.296653 32439 fs_manager.cc:730] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
08:03:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.296772 32439 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data,/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/wal
08:03:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6bb62b6d2754b549b8a04b14f7b06d1"
08:03:38.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:38 on dist-test-slave-x05k"
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.296909 32439 fs_report.cc:389] FS layout report
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/wal
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/wal
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data/data
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.331164 32439 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.331887 32439 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.332105 32439 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.334832 32439 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.334937 32439 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.335047 32439 ts_tablet_manager.cc:610] Registered 0 tablets
08:03:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.335127 32439 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.391438 32439 rpc_server.cc:307] RPC server started. Bound to: 127.31.25.3:34469
08:03:38.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.391475 32574 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.25.3:34469 every 8 connection(s)
08:03:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.392652 32439 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/data/info.pb
08:03:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.394999 32576 heartbeater.cc:344] Connected to a master server at 127.31.25.62:42415
08:03:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.396529 32576 heartbeater.cc:461] Registering TS with master...
08:03:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.397565 32576 heartbeater.cc:507] Master 127.31.25.62:42415 requested a full tablet report, sending...
08:03:38.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.399256 31886 ts_manager.cc:194] Registered new tserver with Master: b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:38.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.401541 32577 heartbeater.cc:344] Connected to a master server at 127.31.25.61:34765
08:03:38.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.401666 32577 heartbeater.cc:461] Registering TS with master...
08:03:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.402019 31844 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu as pid 32439
08:03:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.402169 31844 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/ts-2/wal/instance
08:03:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.402145 32577 heartbeater.cc:507] Master 127.31.25.61:34765 requested a full tablet report, sending...
08:03:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.403085 31966 ts_manager.cc:194] Registered new tserver with Master: b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.404086 31966 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.25.3:52159
08:03:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.404742 32575 heartbeater.cc:344] Connected to a master server at 127.31.25.60:44905
08:03:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.404879 32575 heartbeater.cc:461] Registering TS with master...
08:03:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.405634 32575 heartbeater.cc:507] Master 127.31.25.60:44905 requested a full tablet report, sending...
08:03:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.406505 32048 ts_manager.cc:194] Registered new tserver with Master: b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.410055 31844 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:03:38.411 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:38.411 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:03:38.412 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.31.25.62"
port: 42415
}
bound_http_address {
host: "127.31.25.62"
port: 38489
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.31.25.61"
port: 34765
}
bound_http_address {
host: "127.31.25.61"
port: 35161
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.31.25.60"
port: 44905
}
bound_http_address {
host: "127.31.25.60"
port: 37693
}
}
}
08:03:38.413 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:03:38.414 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.31.25.1"
port: 46853
}
bound_http_address {
host: "127.31.25.1"
port: 44879
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.31.25.2"
port: 33279
}
bound_http_address {
host: "127.31.25.2"
port: 37577
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.31.25.3"
port: 34469
}
bound_http_address {
host: "127.31.25.3"
port: 41567
}
}
}
08:03:38.414 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:38.416 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:03:38.418 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.25.60:44905(127.31.25.60:44905)] connecting to peer
08:03:38.419 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:38.419 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.25.61:34765(127.31.25.61:34765)] connecting to peer
08:03:38.421 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.31.25.62:42415(127.31.25.62:42415)] Successfully connected to peer
08:03:38.422 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.31.25.60:44905(127.31.25.60:44905)] Successfully connected to peer
08:03:38.423 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.31.25.61:34765(127.31.25.61:34765)] Successfully connected to peer
08:03:38.423 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.31.25.60
08:03:38.423 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.31.25.62
08:03:38.427 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.31.25.61
08:03:38.435 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6440f8a9, L:/127.0.0.1:44188 - R:/127.31.25.62:42415] using SASL/PLAIN
08:03:38.441 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4033b831, L:/127.0.0.1:42548 - R:/127.31.25.61:34765] using SASL/PLAIN
08:03:38.442 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6ff43ca5, L:/127.0.0.1:58738 - R:/127.31.25.60:44905] using SASL/PLAIN
08:03:38.443 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:38.444 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=26), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [20ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, deferred=null) after lookup
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.445439 31966 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42548:
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
08:03:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:38.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.447254 31966 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:38.476 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ee98af9fd6e749bf8a1e7c0668e17162
08:03:38.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.525439 32295 heartbeater.cc:499] Master 127.31.25.61:34765 was elected leader, sending a full tablet report...
08:03:38.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.525733 32369 tablet_service.cc:1468] Processing CreateTablet for tablet 8d7bc5616ccd41a893fc08f0541f97c2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:38.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.527757 32369 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d7bc5616ccd41a893fc08f0541f97c2. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:38.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.526775 32368 tablet_service.cc:1468] Processing CreateTablet for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:38.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.528733 32368 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:38.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.534982 32222 tablet_service.cc:1468] Processing CreateTablet for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.535576 32510 tablet_service.cc:1468] Processing CreateTablet for tablet 8d7bc5616ccd41a893fc08f0541f97c2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.537041 32222 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.537106 32510 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d7bc5616ccd41a893fc08f0541f97c2. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:38.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.536347 32509 tablet_service.cc:1468] Processing CreateTablet for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
08:03:38.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.537655 32509 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:38.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.536175 32223 tablet_service.cc:1468] Processing CreateTablet for tablet 8d7bc5616ccd41a893fc08f0541f97c2 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
08:03:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.542061 32223 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8d7bc5616ccd41a893fc08f0541f97c2. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:38.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.550318 32600 tablet_bootstrap.cc:492] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629: Bootstrap starting.
08:03:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.552980 32601 tablet_bootstrap.cc:492] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1: Bootstrap starting.
08:03:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.553253 32602 tablet_bootstrap.cc:492] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e: Bootstrap starting.
08:03:38.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.554214 32600 tablet_bootstrap.cc:654] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629: Neither blocks nor log segments found. Creating new log.
08:03:38.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.555548 32601 tablet_bootstrap.cc:654] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1: Neither blocks nor log segments found. Creating new log.
08:03:38.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.555541 32600 log.cc:826] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629: Log is configured to *not* fsync() on all Append() calls
08:03:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.556361 32602 tablet_bootstrap.cc:654] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e: Neither blocks nor log segments found. Creating new log.
08:03:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.556695 32601 log.cc:826] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1: Log is configured to *not* fsync() on all Append() calls
08:03:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.557379 32602 log.cc:826] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e: Log is configured to *not* fsync() on all Append() calls
08:03:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.558166 32600 tablet_bootstrap.cc:492] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629: No bootstrap required, opened a new log
08:03:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.558885 32600 ts_tablet_manager.cc:1397] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.004s
08:03:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.559494 32601 tablet_bootstrap.cc:492] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1: No bootstrap required, opened a new log
08:03:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.559736 32601 ts_tablet_manager.cc:1397] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
08:03:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.559827 32602 tablet_bootstrap.cc:492] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e: No bootstrap required, opened a new log
08:03:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.560047 32602 ts_tablet_manager.cc:1397] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
08:03:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.564297 32602 raft_consensus.cc:357] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.564561 32602 raft_consensus.cc:383] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.564667 32602 raft_consensus.cc:738] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81065a6e8be5478eab56c94b4058873e, State: Initialized, Role: FOLLOWER
08:03:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.565179 32602 consensus_queue.cc:260] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.565299 32600 raft_consensus.cc:357] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.565707 32600 raft_consensus.cc:383] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.565891 32600 raft_consensus.cc:738] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c99bc40bf5164ce5949dc5c176536629, State: Initialized, Role: FOLLOWER
08:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.565869 32601 raft_consensus.cc:357] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.566246 32601 raft_consensus.cc:383] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.566416 32601 raft_consensus.cc:738] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b6bb62b6d2754b549b8a04b14f7b06d1, State: Initialized, Role: FOLLOWER
08:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.566563 32602 ts_tablet_manager.cc:1428] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e: Time spent starting tablet: real 0.006s user 0.007s sys 0.000s
08:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.566551 32600 consensus_queue.cc:260] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.567924 32601 consensus_queue.cc:260] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.572635 32602 tablet_bootstrap.cc:492] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e: Bootstrap starting.
08:03:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.574143 32602 tablet_bootstrap.cc:654] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e: Neither blocks nor log segments found. Creating new log.
08:03:38.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.585455 32602 tablet_bootstrap.cc:492] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e: No bootstrap required, opened a new log
08:03:38.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.585625 32602 ts_tablet_manager.cc:1397] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.007s
08:03:38.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.585763 32601 ts_tablet_manager.cc:1428] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1: Time spent starting tablet: real 0.026s user 0.008s sys 0.003s
08:03:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.586397 32602 raft_consensus.cc:357] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.586592 32602 raft_consensus.cc:383] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.586660 32602 raft_consensus.cc:738] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81065a6e8be5478eab56c94b4058873e, State: Initialized, Role: FOLLOWER
08:03:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.586910 32602 consensus_queue.cc:260] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.587271 32602 ts_tablet_manager.cc:1428] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
08:03:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.593356 32577 heartbeater.cc:499] Master 127.31.25.61:34765 was elected leader, sending a full tablet report...
08:03:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.596113 32601 tablet_bootstrap.cc:492] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1: Bootstrap starting.
08:03:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.602352 32601 tablet_bootstrap.cc:654] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1: Neither blocks nor log segments found. Creating new log.
08:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.603978 32607 raft_consensus.cc:491] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.604441 32436 heartbeater.cc:499] Master 127.31.25.61:34765 was elected leader, sending a full tablet report...
08:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.604337 32607 raft_consensus.cc:513] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.605044 32600 ts_tablet_manager.cc:1428] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629: Time spent starting tablet: real 0.046s user 0.012s sys 0.032s
08:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.605502 32600 tablet_bootstrap.cc:492] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629: Bootstrap starting.
08:03:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.607532 32600 tablet_bootstrap.cc:654] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629: Neither blocks nor log segments found. Creating new log.
08:03:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.607762 32607 leader_election.cc:290] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853), b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:38.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.612097 32601 tablet_bootstrap.cc:492] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1: No bootstrap required, opened a new log
08:03:38.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.612411 32601 ts_tablet_manager.cc:1397] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1: Time spent bootstrapping tablet: real 0.016s user 0.007s sys 0.000s
08:03:38.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.613444 32601 raft_consensus.cc:357] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.613798 32601 raft_consensus.cc:383] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:38.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.614459 32601 raft_consensus.cc:738] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b6bb62b6d2754b549b8a04b14f7b06d1, State: Initialized, Role: FOLLOWER
08:03:38.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.614894 32601 consensus_queue.cc:260] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.618712 32601 ts_tablet_manager.cc:1428] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
08:03:38.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.619685 32248 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "94eee9dbef6f47fa9f119a0ce6a9e0a5" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81065a6e8be5478eab56c94b4058873e" is_pre_election: true
08:03:38.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.620258 32248 raft_consensus.cc:2466] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 0.
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.621573 32313 leader_election.cc:304] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 81065a6e8be5478eab56c94b4058873e, c99bc40bf5164ce5949dc5c176536629; no voters:
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.622041 32600 tablet_bootstrap.cc:492] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629: No bootstrap required, opened a new log
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.621992 32607 raft_consensus.cc:2802] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.622182 32607 raft_consensus.cc:491] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.622229 32600 ts_tablet_manager.cc:1397] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629: Time spent bootstrapping tablet: real 0.017s user 0.007s sys 0.003s
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.622284 32607 raft_consensus.cc:3058] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Advancing to term 1
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.623194 32600 raft_consensus.cc:357] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.623411 32600 raft_consensus.cc:383] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.623504 32600 raft_consensus.cc:738] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c99bc40bf5164ce5949dc5c176536629, State: Initialized, Role: FOLLOWER
08:03:38.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.623762 32600 consensus_queue.cc:260] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.624260 32600 ts_tablet_manager.cc:1428] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
08:03:38.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.626360 32607 raft_consensus.cc:513] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.627010 32607 leader_election.cc:290] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 election: Requested vote from peers 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853), b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.626924 32530 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "94eee9dbef6f47fa9f119a0ce6a9e0a5" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" is_pre_election: true
08:03:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.627352 32530 raft_consensus.cc:2466] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 0.
08:03:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.627337 32248 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "94eee9dbef6f47fa9f119a0ce6a9e0a5" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81065a6e8be5478eab56c94b4058873e"
08:03:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.627496 32248 raft_consensus.cc:3058] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Advancing to term 1
08:03:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.627671 32529 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "94eee9dbef6f47fa9f119a0ce6a9e0a5" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1"
08:03:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.627867 32529 raft_consensus.cc:3058] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Advancing to term 1
08:03:38.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.629424 32248 raft_consensus.cc:2466] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P 81065a6e8be5478eab56c94b4058873e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 1.
08:03:38.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.629666 32529 raft_consensus.cc:2466] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 1.
08:03:38.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.629781 32313 leader_election.cc:304] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 81065a6e8be5478eab56c94b4058873e, c99bc40bf5164ce5949dc5c176536629; no voters:
08:03:38.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.630097 32607 raft_consensus.cc:2802] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 1 FOLLOWER]: Leader election won for term 1
08:03:38.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.630236 32607 raft_consensus.cc:695] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [term 1 LEADER]: Becoming Leader. State: Replica: c99bc40bf5164ce5949dc5c176536629, State: Running, Role: LEADER
08:03:38.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.630700 32607 consensus_queue.cc:237] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.635079 31966 catalog_manager.cc:5582] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 reported cstate change: term changed from 0 to 1, leader changed from <none> to c99bc40bf5164ce5949dc5c176536629 (127.31.25.2). New cstate: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } health_report { overall_health: UNKNOWN } } }
08:03:38.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.645968 32578 tablet.cc:2378] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.703825 32437 tablet.cc:2378] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:38.770843 32296 tablet.cc:2378] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:38.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.991037 32606 raft_consensus.cc:491] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:38.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.991245 32606 raft_consensus.cc:513] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:38.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.992681 32606 leader_election.cc:290] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279), b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:38.999821 32529 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8d7bc5616ccd41a893fc08f0541f97c2" candidate_uuid: "81065a6e8be5478eab56c94b4058873e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" is_pre_election: true
08:03:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.000061 32529 raft_consensus.cc:2466] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 81065a6e8be5478eab56c94b4058873e in term 0.
08:03:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.000057 32389 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8d7bc5616ccd41a893fc08f0541f97c2" candidate_uuid: "81065a6e8be5478eab56c94b4058873e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c99bc40bf5164ce5949dc5c176536629" is_pre_election: true
08:03:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.000401 32389 raft_consensus.cc:2466] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 81065a6e8be5478eab56c94b4058873e in term 0.
08:03:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.000559 32172 leader_election.cc:304] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 81065a6e8be5478eab56c94b4058873e, b6bb62b6d2754b549b8a04b14f7b06d1; no voters:
08:03:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.000943 32606 raft_consensus.cc:2802] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.001087 32606 raft_consensus.cc:491] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:39.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.001180 32606 raft_consensus.cc:3058] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Advancing to term 1
08:03:39.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.003132 32606 raft_consensus.cc:513] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.003666 32606 leader_election.cc:290] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [CANDIDATE]: Term 1 election: Requested vote from peers c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279), b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.004062 32389 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8d7bc5616ccd41a893fc08f0541f97c2" candidate_uuid: "81065a6e8be5478eab56c94b4058873e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c99bc40bf5164ce5949dc5c176536629"
08:03:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.004194 32389 raft_consensus.cc:3058] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Advancing to term 1
08:03:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.004137 32529 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "8d7bc5616ccd41a893fc08f0541f97c2" candidate_uuid: "81065a6e8be5478eab56c94b4058873e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1"
08:03:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.004334 32529 raft_consensus.cc:3058] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Advancing to term 1
08:03:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.005867 32389 raft_consensus.cc:2466] T 8d7bc5616ccd41a893fc08f0541f97c2 P c99bc40bf5164ce5949dc5c176536629 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 81065a6e8be5478eab56c94b4058873e in term 1.
08:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.005923 32529 raft_consensus.cc:2466] T 8d7bc5616ccd41a893fc08f0541f97c2 P b6bb62b6d2754b549b8a04b14f7b06d1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 81065a6e8be5478eab56c94b4058873e in term 1.
08:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.006201 32172 leader_election.cc:304] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 81065a6e8be5478eab56c94b4058873e, c99bc40bf5164ce5949dc5c176536629; no voters:
08:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.006435 32606 raft_consensus.cc:2802] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 1 FOLLOWER]: Leader election won for term 1
08:03:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.006825 32606 raft_consensus.cc:695] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [term 1 LEADER]: Becoming Leader. State: Replica: 81065a6e8be5478eab56c94b4058873e, State: Running, Role: LEADER
08:03:39.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.007318 32606 consensus_queue.cc:237] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.011857 31966 catalog_manager.cc:5582] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e reported cstate change: term changed from 0 to 1, leader changed from <none> to 81065a6e8be5478eab56c94b4058873e (127.31.25.1). New cstate: current_term: 1 leader_uuid: "81065a6e8be5478eab56c94b4058873e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } health_report { overall_health: UNKNOWN } } }
08:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.019299 32607 consensus_queue.cc:1035] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.027590 32607 consensus_queue.cc:1035] T 94eee9dbef6f47fa9f119a0ce6a9e0a5 P c99bc40bf5164ce5949dc5c176536629 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.258316 31967 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42560:
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:39.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:39.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.259472 31967 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.270141 32222 tablet_service.cc:1468] Processing CreateTablet for tablet 0e68f001cc84451980012724f8b2131d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=2abe0f70ce32440c88eef84269a9bb11]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.270633 32368 tablet_service.cc:1468] Processing CreateTablet for tablet 0e68f001cc84451980012724f8b2131d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=2abe0f70ce32440c88eef84269a9bb11]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.271142 32222 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e68f001cc84451980012724f8b2131d. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.270848 32509 tablet_service.cc:1468] Processing CreateTablet for tablet 0e68f001cc84451980012724f8b2131d (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=2abe0f70ce32440c88eef84269a9bb11]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
08:03:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.271432 32368 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e68f001cc84451980012724f8b2131d. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.271461 32509 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0e68f001cc84451980012724f8b2131d. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.276520 32638 tablet_bootstrap.cc:492] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: Bootstrap starting.
08:03:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.277626 32639 tablet_bootstrap.cc:492] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: Bootstrap starting.
08:03:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.278013 32638 tablet_bootstrap.cc:654] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: Neither blocks nor log segments found. Creating new log.
08:03:39.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.279626 32639 tablet_bootstrap.cc:654] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: Neither blocks nor log segments found. Creating new log.
08:03:39.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.279772 32640 tablet_bootstrap.cc:492] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: Bootstrap starting.
08:03:39.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.281471 32640 tablet_bootstrap.cc:654] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: Neither blocks nor log segments found. Creating new log.
08:03:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.281934 32639 tablet_bootstrap.cc:492] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: No bootstrap required, opened a new log
08:03:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.282069 32639 ts_tablet_manager.cc:1397] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
08:03:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.282074 32638 tablet_bootstrap.cc:492] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: No bootstrap required, opened a new log
08:03:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.282200 32638 ts_tablet_manager.cc:1397] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.003s
08:03:39.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.282688 32639 raft_consensus.cc:357] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.282896 32639 raft_consensus.cc:383] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283005 32639 raft_consensus.cc:738] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c99bc40bf5164ce5949dc5c176536629, State: Initialized, Role: FOLLOWER
08:03:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283145 32638 raft_consensus.cc:357] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283247 32639 consensus_queue.cc:260] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283360 32638 raft_consensus.cc:383] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283450 32638 raft_consensus.cc:738] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b6bb62b6d2754b549b8a04b14f7b06d1, State: Initialized, Role: FOLLOWER
08:03:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283699 32639 ts_tablet_manager.cc:1428] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
08:03:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283733 32623 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.283967 32623 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:39.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.286288 32638 consensus_queue.cc:260] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.287915 32608 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.288210 32608 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.289875 32638 ts_tablet_manager.cc:1428] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
08:03:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.291760 32640 tablet_bootstrap.cc:492] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: No bootstrap required, opened a new log
08:03:39.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.291906 32640 ts_tablet_manager.cc:1397] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.000s
08:03:39.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.292483 32640 raft_consensus.cc:357] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.292627 32640 raft_consensus.cc:383] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:39.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.292698 32640 raft_consensus.cc:738] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 81065a6e8be5478eab56c94b4058873e, State: Initialized, Role: FOLLOWER
08:03:39.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.292925 32640 consensus_queue.cc:260] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.293363 32640 ts_tablet_manager.cc:1428] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
08:03:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.293293 32606 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.293519 32606 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:39.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.395486 32623 raft_consensus.cc:491] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:39.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.395651 32623 raft_consensus.cc:513] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.396315 32623 leader_election.cc:290] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853), b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.396809 32248 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0e68f001cc84451980012724f8b2131d" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81065a6e8be5478eab56c94b4058873e" is_pre_election: true
08:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.396831 32529 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0e68f001cc84451980012724f8b2131d" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" is_pre_election: true
08:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.397045 32248 raft_consensus.cc:2466] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 0.
08:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.397043 32529 raft_consensus.cc:2466] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 0.
08:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.397622 32316 leader_election.cc:304] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b6bb62b6d2754b549b8a04b14f7b06d1, c99bc40bf5164ce5949dc5c176536629; no voters:
08:03:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.398562 32623 raft_consensus.cc:2802] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.398726 32623 raft_consensus.cc:491] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.398886 32623 raft_consensus.cc:3058] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 0 FOLLOWER]: Advancing to term 1
08:03:39.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.401597 32623 raft_consensus.cc:513] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.402153 32623 leader_election.cc:290] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 election: Requested vote from peers 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853), b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.402619 32529 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0e68f001cc84451980012724f8b2131d" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1"
08:03:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.402740 32248 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "0e68f001cc84451980012724f8b2131d" candidate_uuid: "c99bc40bf5164ce5949dc5c176536629" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "81065a6e8be5478eab56c94b4058873e"
08:03:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.402931 32529 raft_consensus.cc:3058] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [term 0 FOLLOWER]: Advancing to term 1
08:03:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.402971 32248 raft_consensus.cc:3058] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [term 0 FOLLOWER]: Advancing to term 1
08:03:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.405308 32529 raft_consensus.cc:2466] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 1.
08:03:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.405673 32316 leader_election.cc:304] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b6bb62b6d2754b549b8a04b14f7b06d1, c99bc40bf5164ce5949dc5c176536629; no voters:
08:03:39.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.405673 32248 raft_consensus.cc:2466] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c99bc40bf5164ce5949dc5c176536629 in term 1.
08:03:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.405941 32623 raft_consensus.cc:2802] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 1 FOLLOWER]: Leader election won for term 1
08:03:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.406373 32623 raft_consensus.cc:695] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [term 1 LEADER]: Becoming Leader. State: Replica: c99bc40bf5164ce5949dc5c176536629, State: Running, Role: LEADER
08:03:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.406574 32623 consensus_queue.cc:237] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } }
08:03:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.407987 32610 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: TxnStatusTablet state changed. Reason: New leader c99bc40bf5164ce5949dc5c176536629. Latest consensus state: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.408193 32610 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: This TxnStatusTablet replica's current role is: LEADER
08:03:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.408718 32645 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
08:03:39.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.408967 32645 txn_status_manager.cc:930] Loading transaction status metadata into memory...
08:03:39.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.410068 31967 catalog_manager.cc:5582] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 reported cstate change: term changed from 0 to 1, leader changed from <none> to c99bc40bf5164ce5949dc5c176536629 (127.31.25.2). New cstate: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } health_report { overall_health: UNKNOWN } } }
08:03:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.437363 32529 raft_consensus.cc:1273] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1 [term 1 FOLLOWER]: Refusing update from remote peer c99bc40bf5164ce5949dc5c176536629: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.437381 32248 raft_consensus.cc:1273] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e [term 1 FOLLOWER]: Refusing update from remote peer c99bc40bf5164ce5949dc5c176536629: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.437919 32610 consensus_queue.cc:1035] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:39.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.438141 32623 consensus_queue.cc:1035] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629 [LEADER]: Connected to new peer: Peer: permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:39.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.443581 32606 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: TxnStatusTablet state changed. Reason: New leader c99bc40bf5164ce5949dc5c176536629. Latest consensus state: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.443804 32606 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:39.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.446431 32622 mvcc.cc:204] Tried to move back new op lower bound from 7196561487609147392 to 7196561487491407872. Current Snapshot: MvccSnapshot[applied={T|T < 7196561487609147392}]
08:03:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.447197 32607 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.447461 32607 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: This TxnStatusTablet replica's current role is: LEADER
08:03:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.448148 32608 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: TxnStatusTablet state changed. Reason: New leader c99bc40bf5164ce5949dc5c176536629. Latest consensus state: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.448318 32608 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.452148 32607 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.452335 32607 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P c99bc40bf5164ce5949dc5c176536629: This TxnStatusTablet replica's current role is: LEADER
08:03:39.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.455842 32619 consensus_queue.cc:1035] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:39.461 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
08:03:39.464 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table ee98af9fd6e749bf8a1e7c0668e17162
08:03:39.464 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.31.25.61:34765
08:03:39.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:03:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.468216 31844 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu pid 31935
08:03:39.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.470295 32007 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:39.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.476234 32606 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.476503 32606 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P 81065a6e8be5478eab56c94b4058873e: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.483752 32619 consensus_queue.cc:1035] T 8d7bc5616ccd41a893fc08f0541f97c2 P 81065a6e8be5478eab56c94b4058873e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.488106 32608 tablet_replica.cc:440] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c99bc40bf5164ce5949dc5c176536629" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81065a6e8be5478eab56c94b4058873e" member_type: VOTER last_known_addr { host: "127.31.25.1" port: 46853 } } peers { permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" member_type: VOTER last_known_addr { host: "127.31.25.2" port: 33279 } } peers { permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" member_type: VOTER last_known_addr { host: "127.31.25.3" port: 34469 } } }
08:03:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.488304 32608 tablet_replica.cc:443] T 0e68f001cc84451980012724f8b2131d P b6bb62b6d2754b549b8a04b14f7b06d1: This TxnStatusTablet replica's current role is: FOLLOWER
08:03:39.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.622473 31844 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu with pid 31935
08:03:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.642567 32034 connection.cc:537] server connection from 127.0.0.1:58692 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.643018 31873 connection.cc:537] server connection from 127.0.0.1:44158 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.643087 32436 heartbeater.cc:646] Failed to heartbeat to 127.31.25.61:34765 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.31.25.61:34765 (error 108)
08:03:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.643647 32295 heartbeater.cc:646] Failed to heartbeat to 127.31.25.61:34765 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.31.25.61:34765 (error 108)
08:03:39.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:39.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.644529 32577 heartbeater.cc:646] Failed to heartbeat to 127.31.25.61:34765 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv got EOF from 127.31.25.61:34765 (error 108)
08:03:39.644 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.31.25.61:34765(127.31.25.61:34765)] handling channelInactive
08:03:39.644 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.61:34765(127.31.25.61:34765)] cleaning up while in state READY due to: connection closed
08:03:39.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.921783 32659 raft_consensus.cc:491] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 277fd7418b324b69bb0f9066c33e1fc2)
08:03:39.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.921960 32659 raft_consensus.cc:513] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:39.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.923079 32659 leader_election.cc:290] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765), 8a7b2fb9732a4d81a34ac1b7f5c3f701 (127.31.25.60:44905)
08:03:39.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.923872 32661 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 277fd7418b324b69bb0f9066c33e1fc2)
08:03:39.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.924131 32057 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "95def791ea5d4693ac39455d37983cb2" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" is_pre_election: true
08:03:39.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.924187 32661 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:39.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.924620 32057 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 95def791ea5d4693ac39455d37983cb2 in term 1.
08:03:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.925081 31873 leader_election.cc:304] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8a7b2fb9732a4d81a34ac1b7f5c3f701, 95def791ea5d4693ac39455d37983cb2; no voters:
08:03:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.925412 32659 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 1 FOLLOWER]: Leader pre-election won for term 2
08:03:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.925654 32659 raft_consensus.cc:491] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 277fd7418b324b69bb0f9066c33e1fc2)
08:03:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.925985 32661 leader_election.cc:290] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415), 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765)
08:03:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.925734 31896 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "95def791ea5d4693ac39455d37983cb2" is_pre_election: true
08:03:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.926292 32659 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 1 FOLLOWER]: Advancing to term 2
08:03:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.926267 32032 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.925422 31870 leader_election.cc:336] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.927564 32032 leader_election.cc:336] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:39.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.928524 32659 raft_consensus.cc:513] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:39.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.928825 31896 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 8a7b2fb9732a4d81a34ac1b7f5c3f701 in current term 2: Already voted for candidate 95def791ea5d4693ac39455d37983cb2 in this term.
08:03:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.929101 32659 leader_election.cc:290] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [CANDIDATE]: Term 2 election: Requested vote from peers 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765), 8a7b2fb9732a4d81a34ac1b7f5c3f701 (127.31.25.60:44905)
08:03:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.929571 32057 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "95def791ea5d4693ac39455d37983cb2" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701"
08:03:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.929761 32057 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 1 FOLLOWER]: Advancing to term 2
08:03:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.929740 32033 leader_election.cc:304] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8a7b2fb9732a4d81a34ac1b7f5c3f701; no voters: 277fd7418b324b69bb0f9066c33e1fc2, 95def791ea5d4693ac39455d37983cb2
08:03:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:39.930358 31870 leader_election.cc:336] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111)
08:03:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.931903 32057 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 95def791ea5d4693ac39455d37983cb2 in term 2.
08:03:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.932055 32661 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:03:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.932309 31873 leader_election.cc:304] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8a7b2fb9732a4d81a34ac1b7f5c3f701, 95def791ea5d4693ac39455d37983cb2; no voters: 277fd7418b324b69bb0f9066c33e1fc2
08:03:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.932590 32659 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 2 FOLLOWER]: Leader election won for term 2
08:03:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.932971 32659 raft_consensus.cc:695] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [term 2 LEADER]: Becoming Leader. State: Replica: 95def791ea5d4693ac39455d37983cb2, State: Running, Role: LEADER
08:03:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.933346 32659 consensus_queue.cc:237] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.934487 32665 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 95def791ea5d4693ac39455d37983cb2. Latest consensus state: current_term: 2 leader_uuid: "95def791ea5d4693ac39455d37983cb2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:39.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.934799 32665 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: This master's current role is: LEADER
08:03:39.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.935393 32666 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:03:39.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.937438 32666 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=2abe0f70ce32440c88eef84269a9bb11]
08:03:39.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.938014 32666 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]
08:03:39.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.939568 32666 tablet_loader.cc:96] loaded metadata for tablet 0e68f001cc84451980012724f8b2131d (table kudu_system.kudu_transactions [id=2abe0f70ce32440c88eef84269a9bb11])
08:03:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.940006 32666 tablet_loader.cc:96] loaded metadata for tablet 8d7bc5616ccd41a893fc08f0541f97c2 (table txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162])
08:03:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.940346 32666 tablet_loader.cc:96] loaded metadata for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5 (table txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162])
08:03:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.940624 32666 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:03:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.941479 32666 catalog_manager.cc:1261] Loaded cluster ID: ee4a44ad2f694eaa87bb14a661cd1fa3
08:03:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.941593 32666 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:03:39.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.942715 32666 catalog_manager.cc:1506] Loading token signing keys...
08:03:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.943737 32666 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2: Loaded TSK: 0
08:03:39.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:39.944605 32666 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:03:39.978 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.61:34765(127.31.25.61:34765)] connecting to peer
08:03:39.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765
08:03:39.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.61:34765(127.31.25.61:34765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765
08:03:39.980 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.31.25.61:34765(127.31.25.61:34765) for tablet Kudu Master: Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765
08:03:39.997 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.61:34765(127.31.25.61:34765)] connecting to peer
08:03:39.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765
08:03:39.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.61:34765(127.31.25.61:34765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765
08:03:39.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.61:34765: Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765
08:03:40.000 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:40.000 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=22), Traces: [0ms] sending RPC to server master-127.31.25.61:34765, [2ms] received response from server master-127.31.25.61:34765: Network error: Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [20ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: Network error: Failed to connect to peer master-127.31.25.61:34765(127.31.25.61:34765): Connection refused: /127.31.25.61:34765, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: OK, deferred=Deferred@1743328930(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/604153547@736f34cb, errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/16472678@1b720822)) after lookup
08:03:40.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:40.026422 31870 consensus_peers.cc:489] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 -> Peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Couldn't send request to peer 277fd7418b324b69bb0f9066c33e1fc2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.042534 31885 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:44200:
08:03:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:40.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:40.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:40.044078 31885 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:40.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.051597 32057 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Refusing update from remote peer 95def791ea5d4693ac39455d37983cb2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
08:03:40.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.052264 32665 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
08:03:40.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.053484 32661 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 95def791ea5d4693ac39455d37983cb2. Latest consensus state: current_term: 2 leader_uuid: "95def791ea5d4693ac39455d37983cb2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:40.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.053736 32661 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: This master's current role is: FOLLOWER
08:03:40.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.054986 32659 sys_catalog.cc:455] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "95def791ea5d4693ac39455d37983cb2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:40.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.055234 32659 sys_catalog.cc:458] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 [sys.catalog]: This master's current role is: LEADER
08:03:40.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.056092 32661 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "95def791ea5d4693ac39455d37983cb2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:40.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.056272 32661 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: This master's current role is: FOLLOWER
08:03:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:40.467388 31870 consensus_peers.cc:489] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 -> Peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Couldn't send request to peer 277fd7418b324b69bb0f9066c33e1fc2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:40.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:40.832028 31870 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111) [suppressed 17 similar messages]
08:03:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:40.914218 31870 consensus_peers.cc:489] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 -> Peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Couldn't send request to peer 277fd7418b324b69bb0f9066c33e1fc2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:03:40.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:40.951202 32294 heartbeater.cc:499] Master 127.31.25.62:42415 was elected leader, sending a full tablet report...
08:03:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:41.380573 31870 consensus_peers.cc:489] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 -> Peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Couldn't send request to peer 277fd7418b324b69bb0f9066c33e1fc2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:03:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:41.493052 32576 heartbeater.cc:499] Master 127.31.25.62:42415 was elected leader, sending a full tablet report...
08:03:41.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:41.552485 32435 heartbeater.cc:499] Master 127.31.25.62:42415 was elected leader, sending a full tablet report...
08:03:41.649 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 8d7bc5616ccd41a893fc08f0541f97c2 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
08:03:41.649 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
08:03:41.649 [DEBUG - kudu-nio-1] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=8d7bc5616ccd41a893fc08f0541f97c2@[81065a6e8be5478eab56c94b4058873e(127.31.25.1:46853)[L],b6bb62b6d2754b549b8a04b14f7b06d1(127.31.25.3:34469),c99bc40bf5164ce5949dc5c176536629(127.31.25.2:33279)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=94eee9dbef6f47fa9f119a0ce6a9e0a5@[81065a6e8be5478eab56c94b4058873e(127.31.25.1:46853),b6bb62b6d2754b549b8a04b14f7b06d1(127.31.25.3:34469),c99bc40bf5164ce5949dc5c176536629(127.31.25.2:33279)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
08:03:41.650 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.31.25.62:42415, [1ms] Sub RPC GetTableLocations: received response from server master-127.31.25.62:42415: OK, deferred=null) after lookup
08:03:41.650 [DEBUG - kudu-nio-1] (Connection.java:748) [peer 81065a6e8be5478eab56c94b4058873e(127.31.25.1:46853)] connecting to peer
08:03:41.651 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 81065a6e8be5478eab56c94b4058873e(127.31.25.1:46853)] Successfully connected to peer
08:03:41.662 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x530c514d, L:/127.0.0.1:46758 - R:/127.31.25.1:46853] using TOKEN/null
08:03:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:41.666040 32214 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.31.25.61:34765)
08:03:41.697 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.31.25.61:34765
08:03:41.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:41.698374 31844 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/logs
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/info.pb
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.61:34765
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.61
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35161
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:41.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
08:03:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
08:03:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:41.737028 31855 master_runner.cc:186] Error getting master registration for 127.31.25.61:34765: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111),
08:03:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:41.737231 31855 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:41.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:41.872014 31870 consensus_peers.cc:489] T 00000000000000000000000000000000 P 95def791ea5d4693ac39455d37983cb2 -> Peer 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765): Couldn't send request to peer 277fd7418b324b69bb0f9066c33e1fc2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.61:34765: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
08:03:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:41.995570 32709 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:03:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:41.995832 32709 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:41.995903 32709 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:03:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.003459 32709 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:03:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.003540 32709 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
08:03:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.003585 32709 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:03:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.003620 32709 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.003669 32709 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:03:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.003705 32709 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.003751 32709 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39881/jwks.json
08:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.012728 32709 master_runner.cc:386] Master server non-default flags:
08:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.25.20:37567
08:03:42.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.25.62:42415,127.31.25.61:34765,127.31.25.60:44905
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.25.61:34765
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39881/jwks.json
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/info.pb
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.25.61
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35161
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/logs
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:42.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:42.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:03:42.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:42.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:24 UTC on 24a791456cd2
08:03:42.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7892
08:03:42.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
08:03:42.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.014241 32709 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.015877 32709 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.023254 32716 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.023322 32715 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.024091 32718 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.025010 32709 server_base.cc:1047] running on GCE node
08:03:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.025682 32709 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:42.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.027112 32709 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:42.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.028338 32709 hybrid_clock.cc:648] HybridClock initialized: now 1756973022028272 us; error 42 us; skew 500 ppm
08:03:42.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.028636 32709 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.030639 32709 webserver.cc:480] Webserver started at http://127.31.25.61:35161/ using document root <none> and password file <none>
08:03:42.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.031337 32709 fs_manager.cc:362] Metadata directory not provided
08:03:42.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.031466 32709 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:42.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.036367 32709 fs_manager.cc:714] Time spent opening directory manager: real 0.003s user 0.004s sys 0.000s
08:03:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.038962 32724 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.040143 32709 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.040298 32709 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data,/tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "277fd7418b324b69bb0f9066c33e1fc2"
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:03:36 on dist-test-slave-x05k"
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.041003 32709 fs_report.cc:389] FS layout report
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/wal
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/data
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.058467 32709 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:42.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.059271 32709 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:42.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.059551 32709 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:42.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.080129 32709 rpc_server.cc:307] RPC server started. Bound to: 127.31.25.61:34765
08:03:42.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.080149 317 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.25.61:34765 every 8 connection(s)
08:03:42.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.081637 32709 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskU_GLRA/test-tmp/mini-kudu-cluster8223542978238974957/master-1/data/info.pb
08:03:42.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.083201 31844 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu as pid 32709
08:03:42.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:42.083 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.31.25.62:42415
08:03:42.083 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:03:42.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.084080 31844 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu pid 31855
08:03:42.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.085255 31926 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:42.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.089186 318 sys_catalog.cc:263] Verifying existing consensus state
08:03:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.092300 318 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Bootstrap starting.
08:03:42.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.108295 318 log.cc:826] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Log is configured to *not* fsync() on all Append() calls
08:03:42.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.120683 318 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:03:42.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.121114 318 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: Bootstrap complete.
08:03:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.127560 318 raft_consensus.cc:357] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.128198 318 raft_consensus.cc:738] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 277fd7418b324b69bb0f9066c33e1fc2, State: Initialized, Role: FOLLOWER
08:03:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.128773 318 consensus_queue.cc:260] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.129621 322 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.130046 322 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.130803 318 sys_catalog.cc:564] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: configured and running, proceeding with master startup.
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.141254 333 catalog_manager.cc:1261] Loaded cluster ID: ee4a44ad2f694eaa87bb14a661cd1fa3
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.141364 333 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: loading cluster ID for follower catalog manager: success
08:03:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.146721 333 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: acquiring CA information for follower catalog manager: success
08:03:42.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.149528 333 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:42.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.239028 31844 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu with pid 31855
08:03:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.261401 339 negotiation.cc:337] Failed RPC negotiation. Trace:
08:03:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.235120 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36836 (local address 127.31.25.61:34765)
08:03:42.260 [ERROR - kudu-nio-1] (Connection.java:452) [peer master-127.31.25.62:42415(127.31.25.62:42415)] unexpected exception from downstream on [id: 0x6440f8a9, L:/127.0.0.1:44188 - R:/127.31.25.62:42415]
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method) ~[?:1.8.0_275]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[?:1.8.0_275]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[?:1.8.0_275]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) ~[?:1.8.0_275]
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132) ~[netty-buffer-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
08:03:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.240161 (+ 5041us) server_negotiation.cc:207] Beginning negotiation
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.240172 (+ 11us) server_negotiation.cc:400] Waiting for connection header
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.240207 (+ 35us) server_negotiation.cc:408] Connection header received
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379)
at io.netty.buffer.PooledByteBuf.setBytes(PooledByteBuf.java:255)
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1132)
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:356)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:151)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.240286 (+ 79us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.240296 (+ 10us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.240581 (+ 285us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.240749 (+ 168us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.259882 (+ 19133us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36836: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:42.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4834,"thread_start_us":5440,"threads_started":1}
08:03:42.262 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state READY due to: [peer master-127.31.25.62:42415(127.31.25.62:42415)] unexpected exception from downstream on [id: 0x6440f8a9, L:/127.0.0.1:44188 - R:/127.31.25.62:42415]
08:03:42.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:42.262 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.31.25.62:42415(127.31.25.62:42415) for tablet Kudu Master: [peer master-127.31.25.62:42415(127.31.25.62:42415)] unexpected exception from downstream on [id: 0x6440f8a9, L:/127.0.0.1:44188 - R:/127.31.25.62:42415]
08:03:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.263065 32726 connection.cc:537] client connection to 127.31.25.62:42415 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:42.264 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.31.25.62:42415(127.31.25.62:42415)] handling channelInactive
08:03:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.263445 32709 master_runner.cc:186] Error getting master registration for 127.31.25.62:42415: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107),
08:03:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.267438 32709 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:03:42.278 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.279 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.61:34765(127.31.25.61:34765)] connecting to peer
08:03:42.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.279 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.31.25.61:34765(127.31.25.61:34765)] Successfully connected to peer
08:03:42.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.280 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.292 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xafcaf1bf, L:/127.0.0.1:36850 - R:/127.31.25.61:34765] using TOKEN/null
08:03:42.295 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.318 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.319 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.337 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.341 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.357 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.358621 322 raft_consensus.cc:491] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:42.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.358954 322 raft_consensus.cc:513] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:42.360 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.361166 322 leader_election.cc:290] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415), 8a7b2fb9732a4d81a34ac1b7f5c3f701 (127.31.25.60:44905)
08:03:42.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.362190 32726 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111)
08:03:42.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.377825 32726 leader_election.cc:336] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111)
08:03:42.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.382721 32057 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "277fd7418b324b69bb0f9066c33e1fc2" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" is_pre_election: true
08:03:42.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.383741 32728 leader_election.cc:304] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 277fd7418b324b69bb0f9066c33e1fc2; no voters: 8a7b2fb9732a4d81a34ac1b7f5c3f701, 95def791ea5d4693ac39455d37983cb2
08:03:42.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.384208 322 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:03:42.399 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=5, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] sending RPC to server master-127.31.25.62:42415, [124ms] received response from server master-127.31.25.62:42415: Network error: [peer master-127.31.25.62:42415(127.31.25.62:42415)] unexpected exception from downstream on [id: 0x6440f8a9, L:/127.0.0.1:44188 - R:/127.31.25.62:42415], [124ms] delaying RPC due to: Network error: [peer master-127.31.25.62:42415(127.31.25.62:42415)] unexpected exception from downstream on [id: 0x6440f8a9, L:/127.0.0.1:44188 - R:/127.31.25.62:42415], [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [141ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [142ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [157ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [157ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [181ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [181ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [181ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [201ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [203ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [203ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [220ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [220ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [222ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, deferred=Deferred@1456957781(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$118/604153547@564e6e4a -> (continuation of Deferred@1884796055 after retry RPC after error@1063880794) -> (continuation of Deferred@1385812771 after retry RPC after error@1996544772) -> (continuation of Deferred@977156735 after retry RPC after error@1231441466) -> (continuation of Deferred@1442483454 after retry RPC after error@617854398), errback=org.apache.kudu.client.KuduTransaction$$Lambda$119/16472678@6ff59e99 -> (continuation of Deferred@1884796055 after retry RPC after error@1063880794) -> (continuation of Deferred@1385812771 after retry RPC after error@1996544772) -> (continuation of Deferred@977156735 after retry RPC after error@1231441466) -> (continuation of Deferred@1442483454 after retry RPC after error@617854398)))
08:03:42.401 [WARN - pool-40-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
08:03:42.418 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.419 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.420 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.438 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.439 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.440 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.457 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.459 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.459 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.478 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.479 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.480 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.480 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.498 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.499 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.502 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.534690 344 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 95def791ea5d4693ac39455d37983cb2)
08:03:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.534931 344 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.535657 344 leader_election.cc:290] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415), 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765)
08:03:42.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.537541 32033 leader_election.cc:336] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111)
08:03:42.538 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.539 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.540 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765; will retry
08:03:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.545281 32751 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "277fd7418b324b69bb0f9066c33e1fc2" is_pre_election: true
08:03:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.545801 32751 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8a7b2fb9732a4d81a34ac1b7f5c3f701 in term 1.
08:03:42.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.546288 32032 leader_election.cc:304] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 277fd7418b324b69bb0f9066c33e1fc2, 8a7b2fb9732a4d81a34ac1b7f5c3f701; no voters: 95def791ea5d4693ac39455d37983cb2
08:03:42.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.546694 344 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Leader pre-election won for term 3
08:03:42.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.546825 344 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 95def791ea5d4693ac39455d37983cb2)
08:03:42.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.546949 344 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 2 FOLLOWER]: Advancing to term 3
08:03:42.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.548799 344 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:42.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.549386 344 leader_election.cc:290] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 3 election: Requested vote from peers 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415), 277fd7418b324b69bb0f9066c33e1fc2 (127.31.25.61:34765)
08:03:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.549868 32751 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "277fd7418b324b69bb0f9066c33e1fc2"
08:03:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.550050 32751 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 1 FOLLOWER]: Advancing to term 3
08:03:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.550794 32033 leader_election.cc:336] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111)
08:03:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.553858 32751 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8a7b2fb9732a4d81a34ac1b7f5c3f701 in term 3.
08:03:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.554215 32032 leader_election.cc:304] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 277fd7418b324b69bb0f9066c33e1fc2, 8a7b2fb9732a4d81a34ac1b7f5c3f701; no voters: 95def791ea5d4693ac39455d37983cb2
08:03:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.554539 344 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 3 FOLLOWER]: Leader election won for term 3
08:03:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.555241 344 raft_consensus.cc:695] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [term 3 LEADER]: Becoming Leader. State: Replica: 8a7b2fb9732a4d81a34ac1b7f5c3f701, State: Running, Role: LEADER
08:03:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.555742 344 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } }
08:03:42.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.557914 348 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8a7b2fb9732a4d81a34ac1b7f5c3f701. Latest consensus state: current_term: 3 leader_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:42.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.558300 348 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: This master's current role is: LEADER
08:03:42.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.559046 351 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:03:42.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.561146 351 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=2abe0f70ce32440c88eef84269a9bb11]
08:03:42.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.561635 351 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162]
08:03:42.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.563123 351 tablet_loader.cc:96] loaded metadata for tablet 0e68f001cc84451980012724f8b2131d (table kudu_system.kudu_transactions [id=2abe0f70ce32440c88eef84269a9bb11])
08:03:42.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.563551 351 tablet_loader.cc:96] loaded metadata for tablet 8d7bc5616ccd41a893fc08f0541f97c2 (table txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162])
08:03:42.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.563920 351 tablet_loader.cc:96] loaded metadata for tablet 94eee9dbef6f47fa9f119a0ce6a9e0a5 (table txn_manager_fallback_rolling [id=ee98af9fd6e749bf8a1e7c0668e17162])
08:03:42.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.564196 351 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:03:42.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.564976 351 catalog_manager.cc:1261] Loaded cluster ID: ee4a44ad2f694eaa87bb14a661cd1fa3
08:03:42.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.565068 351 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:03:42.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.566140 351 catalog_manager.cc:1506] Loading token signing keys...
08:03:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.566916 351 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701: Loaded TSK: 0
08:03:42.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.567623 351 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:03:42.598 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.31.25.62:42415(127.31.25.62:42415)] connecting to peer
08:03:42.599 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.599 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.62:42415(127.31.25.62:42415)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.25.62:42415: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415
08:03:42.601 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:03:42.601 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [1ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [1ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [40ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [83ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [181ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, deferred=Deferred@1421973960(state=PENDING, result=null, callback=(continuation of Deferred@180364183 after retry RPC after error@1983545646) -> (continuation of Deferred@2009598188 after retry RPC after error@790609924) -> (continuation of Deferred@608731883 after retry RPC after error@114906301) -> (continuation of Deferred@836875234 after retry RPC after error@1613716026) -> (continuation of Deferred@9491214 after retry RPC after error@1277166365) -> (continuation of Deferred@1893914584 after retry RPC after error@1778826612), errback=(continuation of Deferred@180364183 after retry RPC after error@1983545646) -> (continuation of Deferred@2009598188 after retry RPC after error@790609924) -> (continuation of Deferred@608731883 after retry RPC after error@114906301) -> (continuation of Deferred@836875234 after retry RPC after error@1613716026) -> (continuation of Deferred@9491214 after retry RPC after error@1277166365) -> (continuation of Deferred@1893914584 after retry RPC after error@1778826612))) after lookup
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.647048 32048 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:34444:
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
08:03:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
08:03:42.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.659142 32577 heartbeater.cc:344] Connected to a master server at 127.31.25.61:34765
08:03:42.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.659180 32033 consensus_peers.cc:489] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 -> Peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Couldn't send request to peer 95def791ea5d4693ac39455d37983cb2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:03:42.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.660917 32048 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
08:03:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.665354 32741 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "b6bb62b6d2754b549b8a04b14f7b06d1" instance_seqno: 1756973018384386) as {username='slave'} at 127.31.25.3:45197; Asking this server to re-register.
08:03:42.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.666637 32436 heartbeater.cc:344] Connected to a master server at 127.31.25.61:34765
08:03:42.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.668073 32577 heartbeater.cc:461] Registering TS with master...
08:03:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.669548 32741 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "c99bc40bf5164ce5949dc5c176536629" instance_seqno: 1756973017936984) as {username='slave'} at 127.31.25.2:46987; Asking this server to re-register.
08:03:42.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.669926 32740 ts_manager.cc:194] Registered new tserver with Master: b6bb62b6d2754b549b8a04b14f7b06d1 (127.31.25.3:34469)
08:03:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.670378 32436 heartbeater.cc:461] Registering TS with master...
08:03:42.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.671566 32751 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [term 3 FOLLOWER]: Refusing update from remote peer 8a7b2fb9732a4d81a34ac1b7f5c3f701: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
08:03:42.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.671922 32740 ts_manager.cc:194] Registered new tserver with Master: c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279)
08:03:42.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.672299 348 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [LEADER]: Connected to new peer: Peer: permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
08:03:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.674724 356 meta_cache.cc:302] tablet 0e68f001cc84451980012724f8b2131d: replica c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279) has failed: Timed out: connection negotiation to 127.31.25.2:33279 for RPC CoordinateTransaction timed out after -0.001s (ON_OUTBOUND_QUEUE)
08:03:42.679 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [1ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [1ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [2ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [21ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [22ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [40ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [41ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [62ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [81ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [83ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [121ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [122ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.31.25.60:44905,127.31.25.62:42415,127.31.25.61:34765) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.60:44905, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.62:42415, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.25.61:34765, [181ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.62:42415: Network error: Failed to connect to peer master-127.31.25.62:42415(127.31.25.62:42415): Connection refused: /127.31.25.62:42415, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.61:34765: OK, [182ms] Sub RPC ConnectToMaster: received response from server master-127.31.25.60:44905: OK, [184ms] sending RPC to server master-127.31.25.60:44905, deferred=Deferred@1421973960(state=PENDING, result=null, callback=(continuation of Deferred@180364183 after retry RPC after error@1983545646) -> (continuation of Deferred@2009598188 after retry RPC after error@790609924) -> (continuation of Deferred@608731883 after retry RPC after error@114906301) -> (continuation of Deferred@836875234 after retry RPC after error@1613716026) -> (continuation of Deferred@9491214 after retry RPC after error@1277166365) -> (continuation of Deferred@1893914584 after retry RPC after error@1778826612) -> (continuation of Deferred@242716110 after retry RPC@91358919), errback=(continuation of Deferred@180364183 after retry RPC after error@1983545646) -> (continuation of Deferred@2009598188 after retry RPC after error@790609924) -> (continuation of Deferred@608731883 after retry RPC after error@114906301) -> (continuation of Deferred@836875234 after retry RPC after error@1613716026) -> (continuation of Deferred@9491214 after retry RPC after error@1277166365) -> (continuation of Deferred@1893914584 after retry RPC after error@1778826612) -> (continuation of Deferred@242716110 after retry RPC@91358919)))
08:03:42.682 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=0, TimeoutTracker(timeout=0, elapsed=3), Traces: [0ms] received response from server master-127.31.25.60:44905: Timed out: Failed to write to server: (no server available): CoordinateTransaction(txn_status_tablet_id: "0e68f001cc84451980012724f8b2131d" op { type: KEEP_TXN_ALIVE txn_id: 0 user: "slave" }) passed its deadline: Timed out: connection negotiation to 127.31.25.2:33279 for RPC CoordinateTransaction timed out after -0.001s (ON_OUTBOUND_QUEUE) (error 0), deferred=null) with no deferred attached!
08:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.680104 32078 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:58738 (request call id 13) took 77 ms (client timeout 67 ms). Trace:
08:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.680222 32078 rpcz_store.cc:269] 0904 08:03:42.602476 (+ 0us) service_pool.cc:168] Inserting onto call queue
08:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.602561 (+ 85us) service_pool.cc:225] Handling call
08:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 08:03:42.680049 (+ 77488us) inbound_call.cc:173] Queueing success response
08:03:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":583,"init.queue_time_us":575,"init.run_cpu_time_us":7652,"init.run_wall_time_us":61955,"thread_start_us":1531,"threads_started":5}
08:03:42.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.694059 32319 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.022s from last keepalive heartbeat (effective timeout is 1.000s)
08:03:42.712 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: c99bc40bf5164ce5949dc5c176536629 (127.31.25.2:33279): transaction ID 0 is not available for further commits: state: ABORT_IN_PROGRESS user: "slave" start_timestamp: 1756973019 last_transition_timestamp: 1756973022
08:03:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.712252 322 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8a7b2fb9732a4d81a34ac1b7f5c3f701. Latest consensus state: current_term: 3 leader_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.712468 322 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.711460 374 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.713593 374 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.715752 374 sys_catalog.cc:455] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.715979 374 sys_catalog.cc:458] T 00000000000000000000000000000000 P 277fd7418b324b69bb0f9066c33e1fc2 [sys.catalog]: This master's current role is: FOLLOWER
08:03:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.717211 32346 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.31.25.61:34765)
08:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.720739 348 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "95def791ea5d4693ac39455d37983cb2" member_type: VOTER last_known_addr { host: "127.31.25.62" port: 42415 } } peers { permanent_uuid: "277fd7418b324b69bb0f9066c33e1fc2" member_type: VOTER last_known_addr { host: "127.31.25.61" port: 34765 } } peers { permanent_uuid: "8a7b2fb9732a4d81a34ac1b7f5c3f701" member_type: VOTER last_known_addr { host: "127.31.25.60" port: 44905 } } }
08:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.720974 348 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 [sys.catalog]: This master's current role is: LEADER
08:03:42.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:42.730540 362 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
08:03:42.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.957417 32293 heartbeater.cc:499] Master 127.31.25.60:44905 was elected leader, sending a full tablet report...
08:03:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.959522 32295 heartbeater.cc:344] Connected to a master server at 127.31.25.61:34765
08:03:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.960786 32740 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "81065a6e8be5478eab56c94b4058873e" instance_seqno: 1756973017491469) as {username='slave'} at 127.31.25.1:48695; Asking this server to re-register.
08:03:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.961344 32295 heartbeater.cc:461] Registering TS with master...
08:03:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:42.962349 32740 ts_manager.cc:194] Registered new tserver with Master: 81065a6e8be5478eab56c94b4058873e (127.31.25.1:46853)
08:03:43.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:43.109340 32033 consensus_peers.cc:489] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 -> Peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Couldn't send request to peer 95def791ea5d4693ac39455d37983cb2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:03:43.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:43.593686 32033 consensus_peers.cc:489] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 -> Peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Couldn't send request to peer 95def791ea5d4693ac39455d37983cb2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
08:03:44.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:44.117672 32033 consensus_peers.cc:489] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 -> Peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Couldn't send request to peer 95def791ea5d4693ac39455d37983cb2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
08:03:44.266 [DEBUG - main] (Connection.java:575) [peer master-127.31.25.62:42415(127.31.25.62:42415)] disconnecting while in state TERMINATED
08:03:44.266 [DEBUG - main] (Connection.java:575) [peer master-127.31.25.61:34765(127.31.25.61:34765)] disconnecting while in state READY
08:03:44.266 [DEBUG - main] (Connection.java:575) [peer 81065a6e8be5478eab56c94b4058873e(127.31.25.1:46853)] disconnecting while in state READY
08:03:44.266 [DEBUG - main] (Connection.java:575) [peer master-127.31.25.60:44905(127.31.25.60:44905)] disconnecting while in state READY
08:03:44.267 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.31.25.60:44905(127.31.25.60:44905)] handling channelInactive
08:03:44.267 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:03:44.268 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.25.60:44905(127.31.25.60:44905)] cleaning up while in state READY due to: connection closed
08:03:44.268 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.31.25.61:34765(127.31.25.61:34765)] handling channelInactive
08:03:44.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.25.61:34765(127.31.25.61:34765)] cleaning up while in state READY due to: connection closed
08:03:44.268 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 81065a6e8be5478eab56c94b4058873e(127.31.25.1:46853)] handling channelInactive
08:03:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.268491 31844 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu pid 32154
08:03:44.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 81065a6e8be5478eab56c94b4058873e(127.31.25.1:46853)] cleaning up while in state READY due to: connection closed
08:03:44.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.275611 32288 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:44.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.368070 31844 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu with pid 32154
08:03:44.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.385646 31844 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu pid 32298
08:03:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.391891 32429 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.498905 32575 heartbeater.cc:499] Master 127.31.25.60:44905 was elected leader, sending a full tablet report...
08:03:44.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.514807 31844 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu with pid 32298
08:03:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:44.533932 32455 connection.cc:537] server connection from 127.31.25.2:39611 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:03:44.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.534366 31844 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu pid 32439
08:03:44.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.540957 32570 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:44.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:03:44.634683 32033 consensus_peers.cc:489] T 00000000000000000000000000000000 P 8a7b2fb9732a4d81a34ac1b7f5c3f701 -> Peer 95def791ea5d4693ac39455d37983cb2 (127.31.25.62:42415): Couldn't send request to peer 95def791ea5d4693ac39455d37983cb2. Status: Network error: Client connection negotiation failed: client connection to 127.31.25.62:42415: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
08:03:44.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.638191 31844 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu with pid 32439
08:03:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.654446 31844 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu pid 32709
08:03:44.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.659986 313 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.759678 31844 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu with pid 32709
08:03:44.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.777966 31844 external_mini_cluster.cc:1699] Attempting to check leaks for /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu pid 32015
08:03:44.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.779040 32088 generic_service.cc:196] Checking for leaks (request via RPC)
08:03:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:03:44.883491 31844 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskU_GLRA/build/asan/bin/kudu with pid 32015
08:03:44.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:03:44Z chronyd exiting
08:03:45.029 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)