03:40:50.548 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:40:50.548 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3o63Jh/test-tmp
03:40:50.548 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:40:50.548 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3o63Jh/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:40:50.550 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:40:50.874 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:40:50.874 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:40:50.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:50.877440 26855 mini_oidc.cc:150] Starting JWKS server
03:40:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:50.880966 26855 webserver.cc:469] Webserver started at http://127.0.0.1:36129/ using document root <none> and password file <none>
03:40:50.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:50.881409 26855 mini_oidc.cc:186] Starting OIDC Discovery server
03:40:50.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:50.883157 26855 webserver.cc:469] Webserver started at http://127.0.0.1:40017/ using document root <none> and password file <none>
03:40:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T03:40:50Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:40:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T03:40:50Z Disabled control of system clock
03:40:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:50.949391 26855 external_mini_cluster.cc:1351] Running /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/wal
03:40:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data
03:40:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/logs
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data/info.pb
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.254:45045
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.254
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36129/jwks.json
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:40:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.266005 26867 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:40:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.266582 26867 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:40:51.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.267077 26867 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:40:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.299536 26867 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:40:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.299841 26867 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:40:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.300012 26867 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:40:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.300209 26867 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:40:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.300418 26867 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:40:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.300608 26867 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:40:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.300756 26867 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36129/jwks.json
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:51.336997 26867 master_runner.cc:386] Master server non-default flags:
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/wal
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.254:45045
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36129/jwks.json
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data/info.pb
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.254
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:40:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/logs
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:22:53 UTC on 5fd53c4cbb9d
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6757
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:40:51.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:51.338464 26867 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:40:51.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:51.340126 26867 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:40:51.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.356083 26876 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:52.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:51.356952 26867 server_base.cc:1048] running on GCE node
03:40:52.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.356158 26873 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:52.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:51.356169 26874 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:52.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.537757 26867 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:40:52.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.540571 26867 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:40:52.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.541996 26867 hybrid_clock.cc:648] HybridClock initialized: now 1750822852541950 us; error 64 us; skew 500 ppm
03:40:52.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.542847 26867 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:40:52.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.556815 26867 webserver.cc:469] Webserver started at http://127.26.57.254:45833/ using document root <none> and password file <none>
03:40:52.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.557781 26867 fs_manager.cc:362] Metadata directory not provided
03:40:52.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.558053 26867 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:40:52.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.558527 26867 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:40:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.563570 26867 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data/instance:
03:40:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fe4841fc4024d5ba14f2def054c512d"
03:40:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:52 on dist-test-slave-fc92"
03:40:52.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.564749 26867 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/wal/instance:
03:40:52.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fe4841fc4024d5ba14f2def054c512d"
03:40:52.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:52 on dist-test-slave-fc92"
03:40:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.572461 26867 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
03:40:52.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.578567 26883 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:40:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.579738 26867 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:40:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.580121 26867 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data,/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/wal
03:40:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4fe4841fc4024d5ba14f2def054c512d"
03:40:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:52 on dist-test-slave-fc92"
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.580483 26867 fs_report.cc:389] FS layout report
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/wal
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/wal
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data/data
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:40:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:40:52.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.651468 26867 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:40:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.653048 26867 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:40:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.653558 26867 kserver.cc:163] Server-wide thread pool size limit: 3276
03:40:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.747077 26867 rpc_server.cc:307] RPC server started. Bound to: 127.26.57.254:45045
03:40:52.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.747156 26944 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.57.254:45045 every 8 connection(s)
03:40:52.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.750007 26867 server_base.cc:1180] Dumped server information to /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/data/info.pb
03:40:52.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.753305 26855 external_mini_cluster.cc:1413] Started /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu as pid 26867
03:40:52.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.753924 26855 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-0/wal/instance
03:40:52.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.757239 26945 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:40:52.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.759456 26855 external_mini_cluster.cc:1351] Running /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/wal
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/logs
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data/info.pb
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.253:39363
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.253
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36129/jwks.json
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:40:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:40:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.775794 26945 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } has no permanent_uuid. Determining permanent_uuid...
03:40:52.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.805094 26945 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } has no permanent_uuid. Determining permanent_uuid...
03:40:52.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:52.807803 26884 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:52.812220 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.253:39363: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.861935 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } attempt: 1
03:40:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:52.866279 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.253:39363: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:52.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:52.934692 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } attempt: 2
03:40:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:52.938964 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.253:39363: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:53.030398 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } attempt: 3
03:40:53.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.034649 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.253:39363: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:53.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.076563 26947 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:40:53.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.077137 26947 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:40:53.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.077634 26947 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:40:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.111932 26947 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:40:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.112237 26947 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:40:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.112430 26947 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:40:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.112625 26947 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:40:53.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.112823 26947 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:40:53.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.113008 26947 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:40:53.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.113149 26947 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36129/jwks.json
03:40:53.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:53.149466 26947 master_runner.cc:386] Master server non-default flags:
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/wal
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.253:39363
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36129/jwks.json
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data/info.pb
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.253
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:53.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/logs
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:22:53 UTC on 5fd53c4cbb9d
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6757
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:40:53.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:53.151274 26947 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:40:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:53.153690 26947 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:40:53.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.168447 26957 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:53.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:53.178068 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } attempt: 4
03:40:53.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.184273 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.253:39363: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:53.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:53.483809 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } attempt: 5
03:40:53.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.495056 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.253:39363: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:54.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.042690 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } attempt: 6
03:40:54.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.073112 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.253:39363: Network error: Client connection negotiation failed: client connection to 127.26.57.253:39363: connect: Connection refused (error 111)
03:40:54.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.570700 26955 debug-util.cc:398] Leaking SignalData structure 0x7b0800015640 after lost signal to thread 26947
03:40:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.707787 26955 kernel_stack_watchdog.cc:198] Thread 26947 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:641 for 401ms:
03:40:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:40:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:40:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:40:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:40:54.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:53.168448 26956 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.713997 26947 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.548s user 0.523s sys 1.019s
03:40:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.714473 26958 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1541 milliseconds
03:40:54.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.714469 26947 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.549s user 0.523s sys 1.020s
03:40:54.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.715942 26959 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:54.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.716010 26947 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:40:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.717242 26947 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:40:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.719872 26947 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:40:54.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.721259 26947 hybrid_clock.cc:648] HybridClock initialized: now 1750822854721221 us; error 55 us; skew 500 ppm
03:40:54.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.722139 26947 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:40:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.728705 26947 webserver.cc:469] Webserver started at http://127.26.57.253:44019/ using document root <none> and password file <none>
03:40:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.729768 26947 fs_manager.cc:362] Metadata directory not provided
03:40:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.730149 26947 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:40:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.730674 26947 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:40:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.735419 26947 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data/instance:
03:40:54.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f379c72e104c409a5a2e6694778a9e"
03:40:54.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:54 on dist-test-slave-fc92"
03:40:54.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.736618 26947 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/wal/instance:
03:40:54.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f379c72e104c409a5a2e6694778a9e"
03:40:54.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:54 on dist-test-slave-fc92"
03:40:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.744663 26947 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
03:40:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.751245 26968 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:40:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.752574 26947 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:40:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.752956 26947 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data,/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/wal
03:40:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f379c72e104c409a5a2e6694778a9e"
03:40:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:54 on dist-test-slave-fc92"
03:40:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.753350 26947 fs_report.cc:389] FS layout report
03:40:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/wal
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/wal
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data/data
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:40:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:40:54.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.805174 26947 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:40:54.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.806820 26947 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:40:54.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.807336 26947 kserver.cc:163] Server-wide thread pool size limit: 3276
03:40:54.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.893815 26947 rpc_server.cc:307] RPC server started. Bound to: 127.26.57.253:39363
03:40:54.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.893931 27029 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.57.253:39363 every 8 connection(s)
03:40:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.896656 26947 server_base.cc:1180] Dumped server information to /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/data/info.pb
03:40:54.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.902086 27030 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:40:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.903229 26855 external_mini_cluster.cc:1413] Started /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu as pid 26947
03:40:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.903584 26855 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-1/wal/instance
03:40:54.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.905596 26855 external_mini_cluster.cc:1351] Running /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/wal
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/logs
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data/info.pb
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.252:35693
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.252
03:40:54.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36129/jwks.json
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
03:40:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
03:40:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.916715 27030 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } has no permanent_uuid. Determining permanent_uuid...
03:40:54.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.944852 27030 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } has no permanent_uuid. Determining permanent_uuid...
03:40:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:54.960814 27030 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } has no permanent_uuid. Determining permanent_uuid...
03:40:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.963467 26970 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:54.967861 27030 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.017397 27030 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 1
03:40:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.021703 27030 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.090166 27030 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 2
03:40:55.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.094475 27030 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.133728 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } attempt: 7
03:40:55.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.146449 26945 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } has no permanent_uuid. Determining permanent_uuid...
03:40:55.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.157068 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.185932 27030 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 3
03:40:55.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.190111 27030 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.215577 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 1
03:40:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.220125 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.238911 27032 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:40:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.239534 27032 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:40:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.240031 27032 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:40:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.271816 27032 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:40:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.272125 27032 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
03:40:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.272285 27032 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:40:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.272485 27032 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:40:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.272691 27032 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:40:55.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.272876 27032 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:40:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.273010 27032 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36129/jwks.json
03:40:55.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.301591 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 2
03:40:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.305963 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.310665 27032 master_runner.cc:386] Master server non-default flags:
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/wal
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.252:35693
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36129/jwks.json
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data/info.pb
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.252
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/logs
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:40:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:40:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:40:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:22:53 UTC on 5fd53c4cbb9d
03:40:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6757
03:40:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:40:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.312381 27032 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:40:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.314271 27032 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:40:55.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.329591 27045 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.333575 27030 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 4
03:40:55.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.338063 27030 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.391398 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 3
03:40:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.395326 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.536082 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 4
03:40:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.540681 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.637567 27030 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 5
03:40:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.642777 27030 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:55.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:55.824378 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 5
03:40:55.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.828866 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.190233 27030 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 6
03:40:56.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:56.197239 27030 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:56.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.381589 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 6
03:40:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:56.389048 26945 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.26.57.252:35693: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:40:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.333276 27047 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:55.329604 27044 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:56.535187 27046 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:40:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.535212 27032 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:40:56.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.540122 27032 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:40:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.543504 27032 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:40:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.544989 27032 hybrid_clock.cc:648] HybridClock initialized: now 1750822856544938 us; error 70 us; skew 500 ppm
03:40:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.545969 27032 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:40:56.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.552563 27032 webserver.cc:469] Webserver started at http://127.26.57.252:33643/ using document root <none> and password file <none>
03:40:56.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.553634 27032 fs_manager.cc:362] Metadata directory not provided
03:40:56.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.553900 27032 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:40:56.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.554440 27032 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:40:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.559144 27032 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data/instance:
03:40:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8af068477b047b4b94dd6f9f3a5dfa0"
03:40:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:56 on dist-test-slave-fc92"
03:40:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.560426 27032 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/wal/instance:
03:40:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8af068477b047b4b94dd6f9f3a5dfa0"
03:40:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:56 on dist-test-slave-fc92"
03:40:56.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.568470 27032 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
03:40:56.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.574819 27058 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:40:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.575982 27032 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:40:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.576424 27032 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data,/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/wal
03:40:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8af068477b047b4b94dd6f9f3a5dfa0"
03:40:56.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:40:56 on dist-test-slave-fc92"
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.576792 27032 fs_report.cc:389] FS layout report
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/wal
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/wal
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data/data
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:40:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:40:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:40:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:40:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:40:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:40:56.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:40:56.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.630206 27032 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:40:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.631799 27032 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:40:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.632330 27032 kserver.cc:163] Server-wide thread pool size limit: 3276
03:40:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.719946 27032 rpc_server.cc:307] RPC server started. Bound to: 127.26.57.252:35693
03:40:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.720031 27119 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.57.252:35693 every 8 connection(s)
03:40:56.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.722891 27032 server_base.cc:1180] Dumped server information to /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/data/info.pb
03:40:56.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.728451 27120 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:40:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.731956 26855 external_mini_cluster.cc:1413] Started /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu as pid 27032
03:40:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.732460 26855 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/master-2/wal/instance
03:40:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.756273 27120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } has no permanent_uuid. Determining permanent_uuid...
03:40:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.785039 27120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } has no permanent_uuid. Determining permanent_uuid...
03:40:56.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.795127 27120 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } has no permanent_uuid. Determining permanent_uuid...
03:40:56.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.812708 27120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0: Bootstrap starting.
03:40:56.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.819922 27120 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0: Neither blocks nor log segments found. Creating new log.
03:40:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.821735 27120 log.cc:826] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0: Log is configured to *not* fsync() on all Append() calls
03:40:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.826571 27120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0: No bootstrap required, opened a new log
03:40:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.845193 27120 raft_consensus.cc:357] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.845906 27120 raft_consensus.cc:383] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:40:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.846123 27120 raft_consensus.cc:738] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8af068477b047b4b94dd6f9f3a5dfa0, State: Initialized, Role: FOLLOWER
03:40:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.846812 27120 consensus_queue.cc:260] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:56.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.849776 27128 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.850724 27128 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: This master's current role is: FOLLOWER
03:40:56.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.851962 27120 sys_catalog.cc:564] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: configured and running, proceeding with master startup.
03:40:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:56.876116 27139 catalog_manager.cc:1560] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:40:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:56.876446 27139 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:40:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:56.887745 27064 tablet.cc:2378] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:40:56.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.873167 27032 master_runner.cc:186] Error getting master registration for 127.26.57.254:45045: OK, instance_id { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" instance_seqno: 1750822852719627 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:40:56.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.905593 27032 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.936331 27128 raft_consensus.cc:491] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:40:56.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.936950 27128 raft_consensus.cc:513] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:56.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:56.940613 27128 leader_election.cc:290] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4fe4841fc4024d5ba14f2def054c512d (127.26.57.254:45045), b4f379c72e104c409a5a2e6694778a9e (127.26.57.253:39363)
03:40:57.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.257958 27030 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 7
03:40:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.292810 27030 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: Bootstrap starting.
03:40:57.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.300601 27030 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: Neither blocks nor log segments found. Creating new log.
03:40:57.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.303082 27030 log.cc:826] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: Log is configured to *not* fsync() on all Append() calls
03:40:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.308928 27030 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: No bootstrap required, opened a new log
03:40:57.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.340709 27030 raft_consensus.cc:357] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:57.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.341624 27030 raft_consensus.cc:383] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:40:57.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.341970 27030 raft_consensus.cc:738] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4f379c72e104c409a5a2e6694778a9e, State: Initialized, Role: FOLLOWER
03:40:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.342916 27030 consensus_queue.cc:260] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.345973 27143 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.349017 27143 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: This master's current role is: FOLLOWER
03:40:57.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.350854 27030 sys_catalog.cc:564] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: configured and running, proceeding with master startup.
03:40:57.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.351154 26995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b4f379c72e104c409a5a2e6694778a9e" is_pre_election: true
03:40:57.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.352294 26995 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a8af068477b047b4b94dd6f9f3a5dfa0 in term 0.
03:40:57.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.352995 26995 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54000 (request call id 1) took 412 ms (client timeout 436 ms). Trace:
03:40:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.353475 26995 rpcz_store.cc:269] 0625 03:40:56.940438 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:40:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:40:56.940577 (+ 139us) service_pool.cc:225] Handling call
03:40:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:40:57.352908 (+412331us) inbound_call.cc:173] Queueing success response
03:40:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:40:57.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.355993 27059 leader_election.cc:304] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a8af068477b047b4b94dd6f9f3a5dfa0, b4f379c72e104c409a5a2e6694778a9e; no voters:
03:40:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.356979 27128 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:40:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.357372 27128 raft_consensus.cc:491] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:40:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.357748 27128 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 0 FOLLOWER]: Advancing to term 1
03:40:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.365617 27128 raft_consensus.cc:513] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.369289 27128 leader_election.cc:290] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [CANDIDATE]: Term 1 election: Requested vote from peers 4fe4841fc4024d5ba14f2def054c512d (127.26.57.254:45045), b4f379c72e104c409a5a2e6694778a9e (127.26.57.253:39363)
03:40:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.370599 26995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b4f379c72e104c409a5a2e6694778a9e"
03:40:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.371412 26995 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 0 FOLLOWER]: Advancing to term 1
03:40:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.376636 27061 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.26.57.254:45045 timed out after 0.436s (SENT)
03:40:57.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.387001 26995 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a8af068477b047b4b94dd6f9f3a5dfa0 in term 1.
03:40:57.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.388592 27059 leader_election.cc:304] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a8af068477b047b4b94dd6f9f3a5dfa0, b4f379c72e104c409a5a2e6694778a9e; no voters:
03:40:57.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.389459 27128 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 1 FOLLOWER]: Leader election won for term 1
03:40:57.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.392452 27128 raft_consensus.cc:695] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [term 1 LEADER]: Becoming Leader. State: Replica: a8af068477b047b4b94dd6f9f3a5dfa0, State: Running, Role: LEADER
03:40:57.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.393576 27128 consensus_queue.cc:237] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.423144 26945 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } attempt: 7
03:40:57.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.431125 27155 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a8af068477b047b4b94dd6f9f3a5dfa0. Latest consensus state: current_term: 1 leader_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.431994 27155 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: This master's current role is: LEADER
03:40:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.449849 27159 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
03:40:57.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.457700 27159 catalog_manager.cc:1486] Initializing Kudu cluster ID...
03:40:57.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.460448 27156 catalog_manager.cc:1560] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:40:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.462931 27156 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:40:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.404084 26947 master_runner.cc:186] Error getting master registration for 127.26.57.254:45045: OK, instance_id { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" instance_seqno: 1750822852719627 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:40:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.469990 26947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:40:57.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.479125 27155 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:40:57.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.479753 26945 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: Bootstrap starting.
03:40:57.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.494201 27143 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: SysCatalogTable state changed. Reason: New leader a8af068477b047b4b94dd6f9f3a5dfa0. Latest consensus state: current_term: 1 leader_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.495308 27143 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: This master's current role is: FOLLOWER
03:40:57.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.499480 27128 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.500273 27128 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: This master's current role is: LEADER
03:40:57.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.507628 27143 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.508387 27143 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: This master's current role is: FOLLOWER
03:40:57.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.519255 26945 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: Neither blocks nor log segments found. Creating new log.
03:40:57.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.523136 26945 log.cc:826] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: Log is configured to *not* fsync() on all Append() calls
03:40:57.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.567332 27159 catalog_manager.cc:1349] Generated new cluster ID: ae79a737d3d24e1a8d8afdc6d0b7ae66
03:40:57.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.567690 27159 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
03:40:57.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.571549 26945 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: No bootstrap required, opened a new log
03:40:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.600522 26945 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.601697 26945 raft_consensus.cc:383] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:40:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.602190 26945 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4fe4841fc4024d5ba14f2def054c512d, State: Initialized, Role: FOLLOWER
03:40:57.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.603411 26945 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:40:57.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.617990 27159 catalog_manager.cc:1372] Generated new certificate authority record
03:40:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.620206 27159 catalog_manager.cc:1506] Loading token signing keys...
03:40:57.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.623406 26945 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: configured and running, proceeding with master startup.
03:40:57.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.623586 26908 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4fe4841fc4024d5ba14f2def054c512d" is_pre_election: true
03:40:57.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.623782 26910 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4fe4841fc4024d5ba14f2def054c512d" is_pre_election: true
03:40:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.628898 26910 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:52638 (request call id 2) took 688 ms (client timeout 436 ms). Trace:
03:40:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.629284 26910 rpcz_store.cc:269] 0625 03:40:56.940238 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:40:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:40:56.940401 (+ 163us) service_pool.cc:225] Handling call
03:40:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:40:57.628812 (+688411us) inbound_call.cc:173] Queueing success response
03:40:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:40:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.624409 26907 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 0 FOLLOWER]: Advancing to term 1
03:40:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.624620 26909 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4fe4841fc4024d5ba14f2def054c512d"
03:40:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.641516 27163 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:40:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.623229 27168 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.643327 27168 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: This master's current role is: FOLLOWER
03:40:57.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:57.664872 26889 tablet.cc:2378] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:40:57.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.672349 27159 catalog_manager.cc:5955] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0: Generated new TSK 0
03:40:57.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.673784 27159 catalog_manager.cc:1516] Initializing in-progress tserver states...
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.707999 26855 external_mini_cluster.cc:1351] Running /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/wal
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/logs
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data/info.pb
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.193:0
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.57.193
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.193
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:40:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:40:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.756820 27128 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.757695 27128 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8af068477b047b4b94dd6f9f3a5dfa0 [sys.catalog]: This master's current role is: LEADER
03:40:57.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.763767 27181 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.764647 27181 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: This master's current role is: FOLLOWER
03:40:57.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.767270 27168 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: SysCatalogTable state changed. Reason: New leader a8af068477b047b4b94dd6f9f3a5dfa0. Latest consensus state: current_term: 1 leader_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:40:57.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.768244 27168 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: This master's current role is: FOLLOWER
03:40:57.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:57.781553 27185 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:40:58.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.232367 27182 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:40:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.233054 27182 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:40:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.234099 27182 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:40:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.293411 27182 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:40:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.294462 27182 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:40:58.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.294785 27182 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:40:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.295042 27182 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:40:58.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.295796 27182 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.57.193
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.361627 27182 tablet_server_runner.cc:78] Tablet server non-default flags:
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/wal
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.193:0
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data/info.pb
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.193
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:40:58.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.57.193
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/logs
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:22:53 UTC on 5fd53c4cbb9d
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6757
03:40:58.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:40:58.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.363533 27182 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:40:58.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.365861 27182 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:40:58.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.386873 27191 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:40:58.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.473289 27156 catalog_manager.cc:1261] Loaded cluster ID: ae79a737d3d24e1a8d8afdc6d0b7ae66
03:40:58.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.473672 27156 catalog_manager.cc:1554] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: loading cluster ID for follower catalog manager: success
03:40:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.496476 27156 catalog_manager.cc:1576] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: acquiring CA information for follower catalog manager: success
03:40:58.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.504721 27156 catalog_manager.cc:1604] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:40:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.818367 27185 catalog_manager.cc:1261] Loaded cluster ID: ae79a737d3d24e1a8d8afdc6d0b7ae66
03:40:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.818763 27185 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: loading cluster ID for follower catalog manager: success
03:40:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.828581 27185 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: acquiring CA information for follower catalog manager: success
03:40:58.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:40:58.847513 27185 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:41:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:59.783999 27190 debug-util.cc:398] Leaking SignalData structure 0x7b08000184c0 after lost signal to thread 27182
03:41:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:01.528610 27190 kernel_stack_watchdog.cc:198] Thread 27182 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:641 for 384ms:
03:41:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:41:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:41:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:41:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:41:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:01.530369 27193 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3130 milliseconds
03:41:01.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:40:58.395956 27192 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:01.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:01.536942 27182 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 3.140s user 0.003s sys 0.005s
03:41:01.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:01.537355 27182 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 3.140s user 0.003s sys 0.005s
03:41:01.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:01.538021 27195 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:01.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.538025 27182 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:41:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.539506 27182 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:41:01.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.546463 27182 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:41:01.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.548337 27182 hybrid_clock.cc:648] HybridClock initialized: now 1750822861548247 us; error 90 us; skew 500 ppm
03:41:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.549250 27182 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:41:01.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.557783 27182 webserver.cc:469] Webserver started at http://127.26.57.193:35727/ using document root <none> and password file <none>
03:41:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.558897 27182 fs_manager.cc:362] Metadata directory not provided
03:41:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.559154 27182 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:41:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.559674 27182 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:41:01.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.566103 27182 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data/instance:
03:41:01.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7535bab0b3954be89838f8ec2858c50e"
03:41:01.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:01 on dist-test-slave-fc92"
03:41:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.567865 27182 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/wal/instance:
03:41:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7535bab0b3954be89838f8ec2858c50e"
03:41:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:01 on dist-test-slave-fc92"
03:41:01.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.576534 27182 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
03:41:01.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.583565 27206 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:41:01.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.584851 27182 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.585498 27182 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data,/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/wal
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7535bab0b3954be89838f8ec2858c50e"
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:01 on dist-test-slave-fc92"
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.586014 27182 fs_report.cc:389] FS layout report
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/wal
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/wal
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data/data
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:41:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:41:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:41:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:41:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.645444 27182 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:41:01.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.647048 27182 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:41:01.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.647586 27182 kserver.cc:163] Server-wide thread pool size limit: 3276
03:41:01.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.667711 27182 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:41:01.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.668357 27182 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
03:41:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.668820 27182 ts_tablet_manager.cc:610] Registered 0 tablets
03:41:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:01.669201 27182 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:41:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.055805 27182 rpc_server.cc:307] RPC server started. Bound to: 127.26.57.193:42417
03:41:02.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.056344 27330 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.57.193:42417 every 8 connection(s)
03:41:02.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.058774 27182 server_base.cc:1180] Dumped server information to /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/data/info.pb
03:41:02.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.060081 26855 external_mini_cluster.cc:1413] Started /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu as pid 27182
03:41:02.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.060899 26855 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-0/wal/instance
03:41:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.090117 26855 external_mini_cluster.cc:1351] Running /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:41:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:41:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/wal
03:41:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data
03:41:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:41:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/logs
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data/info.pb
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:41:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.194:0
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.57.194
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.194
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:41:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:41:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.129829 27331 heartbeater.cc:344] Connected to a master server at 127.26.57.252:35693
03:41:02.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.131846 27331 heartbeater.cc:461] Registering TS with master...
03:41:02.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.133496 27331 heartbeater.cc:507] Master 127.26.57.252:35693 requested a full tablet report, sending...
03:41:02.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.139292 27335 heartbeater.cc:344] Connected to a master server at 127.26.57.254:45045
03:41:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.142058 27335 heartbeater.cc:461] Registering TS with master...
03:41:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.143043 27335 heartbeater.cc:507] Master 127.26.57.254:45045 requested a full tablet report, sending...
03:41:02.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.142027 27075 ts_manager.cc:194] Registered new tserver with Master: 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417)
03:41:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.146872 26900 ts_manager.cc:194] Registered new tserver with Master: 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417)
03:41:02.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.149766 27075 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.26.57.193:45703
03:41:02.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.152689 27333 heartbeater.cc:344] Connected to a master server at 127.26.57.253:39363
03:41:02.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.153118 27333 heartbeater.cc:461] Registering TS with master...
03:41:02.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.154127 27333 heartbeater.cc:507] Master 127.26.57.253:39363 requested a full tablet report, sending...
03:41:02.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.157696 26985 ts_manager.cc:194] Registered new tserver with Master: 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417)
03:41:02.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.465044 27332 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:41:02.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.465605 27332 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:41:02.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.466279 27332 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:41:02.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.504480 27332 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:41:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.505306 27332 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:41:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.505604 27332 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.505861 27332 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:41:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.506418 27332 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.57.194
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.546805 27332 tablet_server_runner.cc:78] Tablet server non-default flags:
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/wal
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.194:0
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data/info.pb
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.194
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:41:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.57.194
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/logs
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:22:53 UTC on 5fd53c4cbb9d
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6757
03:41:02.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:41:02.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.548734 27332 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:41:02.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:02.550674 27332 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:41:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.575104 27342 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.161041 27331 heartbeater.cc:499] Master 127.26.57.252:35693 was elected leader, sending a full tablet report...
03:41:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.578155 27345 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:02.575389 27343 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:03.789757 27344 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:41:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.789925 27332 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:41:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.794613 27332 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:41:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.796864 27332 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:41:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.798271 27332 hybrid_clock.cc:648] HybridClock initialized: now 1750822863798228 us; error 77 us; skew 500 ppm
03:41:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.799125 27332 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:41:03.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.806223 27332 webserver.cc:469] Webserver started at http://127.26.57.194:33851/ using document root <none> and password file <none>
03:41:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.807188 27332 fs_manager.cc:362] Metadata directory not provided
03:41:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.807396 27332 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:41:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.807804 27332 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:41:03.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.812458 27332 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data/instance:
03:41:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93d76ef06b6f4b02971cfe687ee8c75a"
03:41:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:03 on dist-test-slave-fc92"
03:41:03.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.813668 27332 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/wal/instance:
03:41:03.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93d76ef06b6f4b02971cfe687ee8c75a"
03:41:03.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:03 on dist-test-slave-fc92"
03:41:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.821043 27332 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
03:41:03.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.826725 27354 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:41:03.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.827952 27332 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
03:41:03.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.828254 27332 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data,/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/wal
03:41:03.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93d76ef06b6f4b02971cfe687ee8c75a"
03:41:03.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:03 on dist-test-slave-fc92"
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.828570 27332 fs_report.cc:389] FS layout report
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/wal
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/wal
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data/data
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:41:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:41:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.885351 27332 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:41:03.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.887056 27332 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:41:03.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.887606 27332 kserver.cc:163] Server-wide thread pool size limit: 3276
03:41:03.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.909044 27332 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:41:03.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.909422 27332 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:41:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.909775 27332 ts_tablet_manager.cc:610] Registered 0 tablets
03:41:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:03.910034 27332 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.002s
03:41:04.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.103744 27332 rpc_server.cc:307] RPC server started. Bound to: 127.26.57.194:43383
03:41:04.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.104223 27478 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.57.194:43383 every 8 connection(s)
03:41:04.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.106400 27332 server_base.cc:1180] Dumped server information to /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/data/info.pb
03:41:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.112061 26855 external_mini_cluster.cc:1413] Started /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu as pid 27332
03:41:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.112524 26855 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-1/wal/instance
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.128345 26855 external_mini_cluster.cc:1351] Running /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/wal
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:41:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/logs
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data/info.pb
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.195:0
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.57.195
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.195
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:41:04.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
03:41:04.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.127790 27479 heartbeater.cc:344] Connected to a master server at 127.26.57.252:35693
03:41:04.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.134259 27479 heartbeater.cc:461] Registering TS with master...
03:41:04.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.135140 27481 heartbeater.cc:344] Connected to a master server at 127.26.57.254:45045
03:41:04.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.135650 27481 heartbeater.cc:461] Registering TS with master...
03:41:04.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.135936 27479 heartbeater.cc:507] Master 127.26.57.252:35693 requested a full tablet report, sending...
03:41:04.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.137369 27481 heartbeater.cc:507] Master 127.26.57.254:45045 requested a full tablet report, sending...
03:41:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.139065 27075 ts_manager.cc:194] Registered new tserver with Master: 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194:43383)
03:41:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.140229 26900 ts_manager.cc:194] Registered new tserver with Master: 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194:43383)
03:41:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.140760 27075 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.26.57.194:55117
03:41:04.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.144610 27480 heartbeater.cc:344] Connected to a master server at 127.26.57.253:39363
03:41:04.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.145016 27480 heartbeater.cc:461] Registering TS with master...
03:41:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.145753 27480 heartbeater.cc:507] Master 127.26.57.253:39363 requested a full tablet report, sending...
03:41:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.148216 26985 ts_manager.cc:194] Registered new tserver with Master: 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194:43383)
03:41:04.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.451239 27484 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:41:04.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.451797 27484 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:41:04.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.452466 27484 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:41:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.486071 27484 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:41:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.486745 27484 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
03:41:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.487010 27484 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
03:41:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.487246 27484 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
03:41:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.487735 27484 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.57.195
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.525734 27484 tablet_server_runner.cc:78] Tablet server non-default flags:
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.57.212:40181
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/wal
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.57.195:0
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data/info.pb
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.57.195
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.57.254:45045,127.26.57.253:39363,127.26.57.252:35693
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.57.195
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/logs
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:41:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:41:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:41:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
03:41:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:41:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 03:22:53 UTC on 5fd53c4cbb9d
03:41:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6757
03:41:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:41:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.527453 27484 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:41:04.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.529349 27484 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:41:04.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.548399 27492 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:05.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.144807 27479 heartbeater.cc:499] Master 127.26.57.252:35693 was elected leader, sending a full tablet report...
03:41:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.549343 27491 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:05.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:04.548439 27494 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:41:05.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:04.550557 27484 server_base.cc:1048] running on GCE node
03:41:05.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.771128 27484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:41:05.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.773975 27484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:41:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.775429 27484 hybrid_clock.cc:648] HybridClock initialized: now 1750822865775384 us; error 60 us; skew 500 ppm
03:41:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.776567 27484 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:41:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.783568 27484 webserver.cc:469] Webserver started at http://127.26.57.195:36405/ using document root <none> and password file <none>
03:41:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.784543 27484 fs_manager.cc:362] Metadata directory not provided
03:41:05.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.784881 27484 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:41:05.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.785328 27484 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
03:41:05.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.790452 27484 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data/instance:
03:41:05.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "253c339569f540d495975da5e18c19ce"
03:41:05.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:05 on dist-test-slave-fc92"
03:41:05.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.791594 27484 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/wal/instance:
03:41:05.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "253c339569f540d495975da5e18c19ce"
03:41:05.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:05 on dist-test-slave-fc92"
03:41:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.799245 27484 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:41:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.804973 27502 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:41:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.806417 27484 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.806720 27484 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data,/tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/wal
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "253c339569f540d495975da5e18c19ce"
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 03:41:05 on dist-test-slave-fc92"
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.807039 27484 fs_report.cc:389] FS layout report
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/wal
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/wal
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data/data
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:41:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:41:05.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.867650 27484 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:41:05.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.869274 27484 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:41:05.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.869704 27484 kserver.cc:163] Server-wide thread pool size limit: 3276
03:41:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.879925 27484 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:41:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.880175 27484 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:41:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.880438 27484 ts_tablet_manager.cc:610] Registered 0 tablets
03:41:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:05.880615 27484 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:41:06.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.079094 27484 rpc_server.cc:307] RPC server started. Bound to: 127.26.57.195:45185
03:41:06.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.079612 27626 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.57.195:45185 every 8 connection(s)
03:41:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.081584 27484 server_base.cc:1180] Dumped server information to /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/data/info.pb
03:41:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.091297 26855 external_mini_cluster.cc:1413] Started /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu as pid 27484
03:41:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.091806 26855 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task3o63Jh/test-tmp/mini-kudu-cluster1601744397741492425/ts-2/wal/instance
03:41:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.091542 27629 heartbeater.cc:344] Connected to a master server at 127.26.57.254:45045
03:41:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.092530 27629 heartbeater.cc:461] Registering TS with master...
03:41:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.093950 27629 heartbeater.cc:507] Master 127.26.57.254:45045 requested a full tablet report, sending...
03:41:06.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.097396 26900 ts_manager.cc:194] Registered new tserver with Master: 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.104382 27627 heartbeater.cc:344] Connected to a master server at 127.26.57.252:35693
03:41:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.104804 27627 heartbeater.cc:461] Registering TS with master...
03:41:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.105419 27628 heartbeater.cc:344] Connected to a master server at 127.26.57.253:39363
03:41:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.105684 27627 heartbeater.cc:507] Master 127.26.57.252:35693 requested a full tablet report, sending...
03:41:06.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.105796 27628 heartbeater.cc:461] Registering TS with master...
03:41:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.107261 27628 heartbeater.cc:507] Master 127.26.57.253:39363 requested a full tablet report, sending...
03:41:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.107957 27075 ts_manager.cc:194] Registered new tserver with Master: 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.110349 26984 ts_manager.cc:194] Registered new tserver with Master: 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.110383 27075 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.26.57.195:52609
03:41:06.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.132573 26855 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
03:41:06.143 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:41:06.143 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:41:06.144 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.26.57.254"
port: 45045
}
bound_http_address {
host: "127.26.57.254"
port: 45833
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.26.57.253"
port: 39363
}
bound_http_address {
host: "127.26.57.253"
port: 44019
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.26.57.252"
port: 35693
}
bound_http_address {
host: "127.26.57.252"
port: 33643
}
}
}
03:41:06.144 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:41:06.145 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.26.57.193"
port: 42417
}
bound_http_address {
host: "127.26.57.193"
port: 35727
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.26.57.194"
port: 43383
}
bound_http_address {
host: "127.26.57.194"
port: 33851
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.26.57.195"
port: 45185
}
bound_http_address {
host: "127.26.57.195"
port: 36405
}
}
}
03:41:06.145 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:41:06.147 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:41:06.149 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:06.150 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.57.253:39363(127.26.57.253:39363)] connecting to peer
03:41:06.151 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.57.254:45045(127.26.57.254:45045)] connecting to peer
03:41:06.152 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.57.252:35693(127.26.57.252:35693)] Successfully connected to peer
03:41:06.153 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.57.253:39363(127.26.57.253:39363)] Successfully connected to peer
03:41:06.154 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.57.254:45045(127.26.57.254:45045)] Successfully connected to peer
03:41:06.155 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.26.57.252
03:41:06.158 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.26.57.254
03:41:06.162 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.26.57.253
03:41:06.192 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0e35cc1f, L:/127.0.0.1:42168 - R:/127.26.57.254:45045] using SASL/PLAIN
03:41:06.193 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xda492022, L:/127.0.0.1:43734 - R:/127.26.57.253:39363] using SASL/PLAIN
03:41:06.194 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa357ce39, L:/127.0.0.1:39294 - R:/127.26.57.252:35693] using SASL/PLAIN
03:41:06.197 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:41:06.197 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=49), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [47ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [48ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: OK, deferred=null) after lookup
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.199291 27075 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39294:
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:41:06.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:06.202601 27075 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:41:06.236 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 97444f6fe8d648a5896ea26c78b964d0
03:41:06.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.292212 27265 tablet_service.cc:1468] Processing CreateTablet for tablet a28195f20b844748a793f5c96e9fd3fd (DEFAULT_TABLE table=txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:41:06.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.294284 27265 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a28195f20b844748a793f5c96e9fd3fd. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.294381 27414 tablet_service.cc:1468] Processing CreateTablet for tablet 73be950dd4ec4cc2ad8aef3302ae5e80 (DEFAULT_TABLE table=txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:41:06.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.295214 27266 tablet_service.cc:1468] Processing CreateTablet for tablet 73be950dd4ec4cc2ad8aef3302ae5e80 (DEFAULT_TABLE table=txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:41:06.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.296355 27413 tablet_service.cc:1468] Processing CreateTablet for tablet a28195f20b844748a793f5c96e9fd3fd (DEFAULT_TABLE table=txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:41:06.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.297966 27266 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 73be950dd4ec4cc2ad8aef3302ae5e80. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.296357 27414 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 73be950dd4ec4cc2ad8aef3302ae5e80. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.301419 27413 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a28195f20b844748a793f5c96e9fd3fd. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.303855 27562 tablet_service.cc:1468] Processing CreateTablet for tablet 73be950dd4ec4cc2ad8aef3302ae5e80 (DEFAULT_TABLE table=txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
03:41:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.305374 27562 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 73be950dd4ec4cc2ad8aef3302ae5e80. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.311781 27561 tablet_service.cc:1468] Processing CreateTablet for tablet a28195f20b844748a793f5c96e9fd3fd (DEFAULT_TABLE table=txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
03:41:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.313061 27561 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a28195f20b844748a793f5c96e9fd3fd. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.338876 27653 tablet_bootstrap.cc:492] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e: Bootstrap starting.
03:41:06.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.341194 27654 tablet_bootstrap.cc:492] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a: Bootstrap starting.
03:41:06.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.348045 27653 tablet_bootstrap.cc:654] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e: Neither blocks nor log segments found. Creating new log.
03:41:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.350031 27654 tablet_bootstrap.cc:654] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a: Neither blocks nor log segments found. Creating new log.
03:41:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.350502 27653 log.cc:826] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e: Log is configured to *not* fsync() on all Append() calls
03:41:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.352449 27654 log.cc:826] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a: Log is configured to *not* fsync() on all Append() calls
03:41:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.352989 27655 tablet_bootstrap.cc:492] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce: Bootstrap starting.
03:41:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.357565 27653 tablet_bootstrap.cc:492] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e: No bootstrap required, opened a new log
03:41:06.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.358664 27654 tablet_bootstrap.cc:492] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a: No bootstrap required, opened a new log
03:41:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.359148 27654 ts_tablet_manager.cc:1397] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a: Time spent bootstrapping tablet: real 0.019s user 0.008s sys 0.007s
03:41:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.359123 27653 ts_tablet_manager.cc:1397] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e: Time spent bootstrapping tablet: real 0.022s user 0.017s sys 0.000s
03:41:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.361258 27655 tablet_bootstrap.cc:654] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce: Neither blocks nor log segments found. Creating new log.
03:41:06.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.363327 27655 log.cc:826] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce: Log is configured to *not* fsync() on all Append() calls
03:41:06.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.368649 27655 tablet_bootstrap.cc:492] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce: No bootstrap required, opened a new log
03:41:06.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.369165 27655 ts_tablet_manager.cc:1397] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce: Time spent bootstrapping tablet: real 0.017s user 0.000s sys 0.012s
03:41:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.387127 27653 raft_consensus.cc:357] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.387123 27654 raft_consensus.cc:357] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.388118 27653 raft_consensus.cc:383] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.388120 27654 raft_consensus.cc:383] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.388438 27653 raft_consensus.cc:738] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7535bab0b3954be89838f8ec2858c50e, State: Initialized, Role: FOLLOWER
03:41:06.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.388430 27654 raft_consensus.cc:738] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 93d76ef06b6f4b02971cfe687ee8c75a, State: Initialized, Role: FOLLOWER
03:41:06.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.389084 27655 raft_consensus.cc:357] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.389394 27653 consensus_queue.cc:260] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.389629 27654 consensus_queue.cc:260] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.390285 27655 raft_consensus.cc:383] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.390570 27655 raft_consensus.cc:738] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 253c339569f540d495975da5e18c19ce, State: Initialized, Role: FOLLOWER
03:41:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.392681 27655 consensus_queue.cc:260] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.395359 27654 ts_tablet_manager.cc:1428] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a: Time spent starting tablet: real 0.036s user 0.031s sys 0.005s
03:41:06.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.397001 27654 tablet_bootstrap.cc:492] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a: Bootstrap starting.
03:41:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.398919 27627 heartbeater.cc:499] Master 127.26.57.252:35693 was elected leader, sending a full tablet report...
03:41:06.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.404333 27654 tablet_bootstrap.cc:654] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a: Neither blocks nor log segments found. Creating new log.
03:41:06.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.405053 27655 ts_tablet_manager.cc:1428] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce: Time spent starting tablet: real 0.036s user 0.027s sys 0.002s
03:41:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.406090 27655 tablet_bootstrap.cc:492] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce: Bootstrap starting.
03:41:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.408238 27660 raft_consensus.cc:491] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:41:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.408823 27660 raft_consensus.cc:513] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.413715 27655 tablet_bootstrap.cc:654] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce: Neither blocks nor log segments found. Creating new log.
03:41:06.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.426954 27660 leader_election.cc:290] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417), 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.439672 27653 ts_tablet_manager.cc:1428] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e: Time spent starting tablet: real 0.080s user 0.034s sys 0.004s
03:41:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.451691 27654 tablet_bootstrap.cc:492] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a: No bootstrap required, opened a new log
03:41:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.452006 27653 tablet_bootstrap.cc:492] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e: Bootstrap starting.
03:41:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.461972 27653 tablet_bootstrap.cc:654] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e: Neither blocks nor log segments found. Creating new log.
03:41:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.463734 27655 tablet_bootstrap.cc:492] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce: No bootstrap required, opened a new log
03:41:06.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.464448 27655 ts_tablet_manager.cc:1397] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce: Time spent bootstrapping tablet: real 0.059s user 0.012s sys 0.027s
03:41:06.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.458236 27654 ts_tablet_manager.cc:1397] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a: Time spent bootstrapping tablet: real 0.056s user 0.025s sys 0.028s
03:41:06.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.469457 27286 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "73be950dd4ec4cc2ad8aef3302ae5e80" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7535bab0b3954be89838f8ec2858c50e" is_pre_election: true
03:41:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.470736 27655 raft_consensus.cc:357] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.471609 27655 raft_consensus.cc:383] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.472028 27655 raft_consensus.cc:738] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 253c339569f540d495975da5e18c19ce, State: Initialized, Role: FOLLOWER
03:41:06.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.471835 27654 raft_consensus.cc:357] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.472573 27654 raft_consensus.cc:383] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.472872 27654 raft_consensus.cc:738] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 93d76ef06b6f4b02971cfe687ee8c75a, State: Initialized, Role: FOLLOWER
03:41:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.473100 27655 consensus_queue.cc:260] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:06.473722 27356 leader_election.cc:343] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417): Illegal state: must be running to vote when last-logged opid is not known
03:41:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.473555 27654 consensus_queue.cc:260] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.476111 27653 tablet_bootstrap.cc:492] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e: No bootstrap required, opened a new log
03:41:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.476568 27653 ts_tablet_manager.cc:1397] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e: Time spent bootstrapping tablet: real 0.025s user 0.005s sys 0.011s
03:41:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.477072 27655 ts_tablet_manager.cc:1428] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce: Time spent starting tablet: real 0.012s user 0.007s sys 0.000s
03:41:06.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.479590 27654 ts_tablet_manager.cc:1428] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a: Time spent starting tablet: real 0.010s user 0.005s sys 0.000s
03:41:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.480826 27653 raft_consensus.cc:357] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.481513 27653 raft_consensus.cc:383] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.481814 27653 raft_consensus.cc:738] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7535bab0b3954be89838f8ec2858c50e, State: Initialized, Role: FOLLOWER
03:41:06.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.482583 27653 consensus_queue.cc:260] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.485949 27653 ts_tablet_manager.cc:1428] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e: Time spent starting tablet: real 0.009s user 0.009s sys 0.000s
03:41:06.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.497519 27582 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "73be950dd4ec4cc2ad8aef3302ae5e80" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "253c339569f540d495975da5e18c19ce" is_pre_election: true
03:41:06.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.498477 27582 raft_consensus.cc:2466] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 93d76ef06b6f4b02971cfe687ee8c75a in term 0.
03:41:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.500114 27356 leader_election.cc:304] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 253c339569f540d495975da5e18c19ce, 93d76ef06b6f4b02971cfe687ee8c75a; no voters: 7535bab0b3954be89838f8ec2858c50e
03:41:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.501168 27660 raft_consensus.cc:2802] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Leader pre-election won for term 1
03:41:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.501529 27660 raft_consensus.cc:491] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:41:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.501866 27660 raft_consensus.cc:3058] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Advancing to term 1
03:41:06.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.507314 27660 raft_consensus.cc:513] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.508780 27660 leader_election.cc:290] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 election: Requested vote from peers 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417), 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.509836 27286 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "73be950dd4ec4cc2ad8aef3302ae5e80" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7535bab0b3954be89838f8ec2858c50e"
03:41:06.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.509922 27582 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "73be950dd4ec4cc2ad8aef3302ae5e80" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "253c339569f540d495975da5e18c19ce"
03:41:06.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.510341 27286 raft_consensus.cc:3058] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Advancing to term 1
03:41:06.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.510401 27582 raft_consensus.cc:3058] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Advancing to term 1
03:41:06.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.515226 27286 raft_consensus.cc:2466] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93d76ef06b6f4b02971cfe687ee8c75a in term 1.
03:41:06.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.515395 27582 raft_consensus.cc:2466] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93d76ef06b6f4b02971cfe687ee8c75a in term 1.
03:41:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.516254 27356 leader_election.cc:304] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7535bab0b3954be89838f8ec2858c50e, 93d76ef06b6f4b02971cfe687ee8c75a; no voters:
03:41:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.516920 27660 raft_consensus.cc:2802] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 FOLLOWER]: Leader election won for term 1
03:41:06.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.518628 27660 raft_consensus.cc:695] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 LEADER]: Becoming Leader. State: Replica: 93d76ef06b6f4b02971cfe687ee8c75a, State: Running, Role: LEADER
03:41:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.519572 27660 consensus_queue.cc:237] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.529160 27073 catalog_manager.cc:5582] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a reported cstate change: term changed from 0 to 1, leader changed from <none> to 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194). New cstate: current_term: 1 leader_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } health_report { overall_health: UNKNOWN } } }
03:41:06.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:06.591841 27630 tablet.cc:2378] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 253c339569f540d495975da5e18c19ce: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:41:06.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.621677 27660 raft_consensus.cc:491] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:41:06.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.622157 27660 raft_consensus.cc:513] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.623780 27660 leader_election.cc:290] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417), 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.624560 27286 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a28195f20b844748a793f5c96e9fd3fd" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7535bab0b3954be89838f8ec2858c50e" is_pre_election: true
03:41:06.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.624799 27582 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a28195f20b844748a793f5c96e9fd3fd" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "253c339569f540d495975da5e18c19ce" is_pre_election: true
03:41:06.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.625196 27286 raft_consensus.cc:2466] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 93d76ef06b6f4b02971cfe687ee8c75a in term 0.
03:41:06.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.625362 27582 raft_consensus.cc:2466] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 93d76ef06b6f4b02971cfe687ee8c75a in term 0.
03:41:06.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.626031 27356 leader_election.cc:304] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7535bab0b3954be89838f8ec2858c50e, 93d76ef06b6f4b02971cfe687ee8c75a; no voters:
03:41:06.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.626616 27660 raft_consensus.cc:2802] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Leader pre-election won for term 1
03:41:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.626886 27660 raft_consensus.cc:491] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:41:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.627159 27660 raft_consensus.cc:3058] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Advancing to term 1
03:41:06.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.631444 27660 raft_consensus.cc:513] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.633081 27660 leader_election.cc:290] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 election: Requested vote from peers 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417), 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.634012 27286 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a28195f20b844748a793f5c96e9fd3fd" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7535bab0b3954be89838f8ec2858c50e"
03:41:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.634191 27582 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "a28195f20b844748a793f5c96e9fd3fd" candidate_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "253c339569f540d495975da5e18c19ce"
03:41:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.634464 27286 raft_consensus.cc:3058] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Advancing to term 1
03:41:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.634603 27582 raft_consensus.cc:3058] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Advancing to term 1
03:41:06.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.638551 27286 raft_consensus.cc:2466] T a28195f20b844748a793f5c96e9fd3fd P 7535bab0b3954be89838f8ec2858c50e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93d76ef06b6f4b02971cfe687ee8c75a in term 1.
03:41:06.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.638667 27582 raft_consensus.cc:2466] T a28195f20b844748a793f5c96e9fd3fd P 253c339569f540d495975da5e18c19ce [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93d76ef06b6f4b02971cfe687ee8c75a in term 1.
03:41:06.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.639592 27356 leader_election.cc:304] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7535bab0b3954be89838f8ec2858c50e, 93d76ef06b6f4b02971cfe687ee8c75a; no voters:
03:41:06.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.640237 27660 raft_consensus.cc:2802] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 FOLLOWER]: Leader election won for term 1
03:41:06.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.640604 27660 raft_consensus.cc:695] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 LEADER]: Becoming Leader. State: Replica: 93d76ef06b6f4b02971cfe687ee8c75a, State: Running, Role: LEADER
03:41:06.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.641296 27660 consensus_queue.cc:237] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:06.642987 27336 tablet.cc:2378] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 7535bab0b3954be89838f8ec2858c50e: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:41:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:06.643086 27483 tablet.cc:2378] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:41:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.649674 27073 catalog_manager.cc:5582] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a reported cstate change: term changed from 0 to 1, leader changed from <none> to 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194). New cstate: current_term: 1 leader_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } health_report { overall_health: UNKNOWN } } }
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.707862 27073 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39300:
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:41:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:06.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:41:06.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:06.710021 27073 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:41:06.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.742202 27265 tablet_service.cc:1468] Processing CreateTablet for tablet d68cd3d0359945dcaa09e7c738153a22 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d48c17c316d549cc9828d9b1ace7f686]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:41:06.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.743968 27265 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d68cd3d0359945dcaa09e7c738153a22. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.749293 27414 tablet_service.cc:1468] Processing CreateTablet for tablet d68cd3d0359945dcaa09e7c738153a22 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d48c17c316d549cc9828d9b1ace7f686]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:41:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.750982 27414 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d68cd3d0359945dcaa09e7c738153a22. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.754629 27561 tablet_service.cc:1468] Processing CreateTablet for tablet d68cd3d0359945dcaa09e7c738153a22 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=d48c17c316d549cc9828d9b1ace7f686]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
03:41:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.755882 27561 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d68cd3d0359945dcaa09e7c738153a22. 1 dirs total, 0 dirs full, 0 dirs failed
03:41:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.773119 27655 tablet_bootstrap.cc:492] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: Bootstrap starting.
03:41:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.778918 27655 tablet_bootstrap.cc:654] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: Neither blocks nor log segments found. Creating new log.
03:41:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.780889 27654 tablet_bootstrap.cc:492] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: Bootstrap starting.
03:41:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.787230 27653 tablet_bootstrap.cc:492] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: Bootstrap starting.
03:41:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.788803 27654 tablet_bootstrap.cc:654] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: Neither blocks nor log segments found. Creating new log.
03:41:06.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.793952 27653 tablet_bootstrap.cc:654] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: Neither blocks nor log segments found. Creating new log.
03:41:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.800016 27655 tablet_bootstrap.cc:492] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: No bootstrap required, opened a new log
03:41:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.800559 27655 ts_tablet_manager.cc:1397] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: Time spent bootstrapping tablet: real 0.028s user 0.007s sys 0.005s
03:41:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.804447 27653 tablet_bootstrap.cc:492] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: No bootstrap required, opened a new log
03:41:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.804267 27655 raft_consensus.cc:357] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.804785 27655 raft_consensus.cc:383] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.804885 27653 ts_tablet_manager.cc:1397] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: Time spent bootstrapping tablet: real 0.018s user 0.010s sys 0.004s
03:41:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.805090 27655 raft_consensus.cc:738] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 253c339569f540d495975da5e18c19ce, State: Initialized, Role: FOLLOWER
03:41:06.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.805830 27655 consensus_queue.cc:260] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.807413 27653 raft_consensus.cc:357] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.807449 27661 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.808069 27653 raft_consensus.cc:383] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.808125 27661 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: This TxnStatusTablet replica's current role is: FOLLOWER
03:41:06.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.808315 27653 raft_consensus.cc:738] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7535bab0b3954be89838f8ec2858c50e, State: Initialized, Role: FOLLOWER
03:41:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.810957 27654 tablet_bootstrap.cc:492] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: No bootstrap required, opened a new log
03:41:06.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.811359 27654 ts_tablet_manager.cc:1397] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: Time spent bootstrapping tablet: real 0.031s user 0.005s sys 0.012s
03:41:06.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.816754 27655 ts_tablet_manager.cc:1428] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: Time spent starting tablet: real 0.016s user 0.006s sys 0.001s
03:41:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.814123 27654 raft_consensus.cc:357] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.819396 27654 raft_consensus.cc:383] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:41:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.819700 27654 raft_consensus.cc:738] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 93d76ef06b6f4b02971cfe687ee8c75a, State: Initialized, Role: FOLLOWER
03:41:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.814762 27653 consensus_queue.cc:260] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.821241 27659 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:06.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.822084 27659 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: This TxnStatusTablet replica's current role is: FOLLOWER
03:41:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.826539 27653 ts_tablet_manager.cc:1428] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: Time spent starting tablet: real 0.021s user 0.007s sys 0.001s
03:41:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.826225 27654 consensus_queue.cc:260] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.827677 27660 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.828393 27660 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: This TxnStatusTablet replica's current role is: FOLLOWER
03:41:06.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.836911 27654 ts_tablet_manager.cc:1428] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: Time spent starting tablet: real 0.025s user 0.006s sys 0.001s
03:41:06.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.850781 27661 raft_consensus.cc:491] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:41:06.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.851275 27661 raft_consensus.cc:513] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.856923 27659 raft_consensus.cc:491] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:41:06.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.857398 27659 raft_consensus.cc:513] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.861187 27661 leader_election.cc:290] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417), 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194:43383)
03:41:06.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.865346 27659 leader_election.cc:290] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194:43383), 253c339569f540d495975da5e18c19ce (127.26.57.195:45185)
03:41:06.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.881320 27286 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d68cd3d0359945dcaa09e7c738153a22" candidate_uuid: "253c339569f540d495975da5e18c19ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7535bab0b3954be89838f8ec2858c50e" is_pre_election: true
03:41:06.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.882052 27286 raft_consensus.cc:2466] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 253c339569f540d495975da5e18c19ce in term 0.
03:41:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.883316 27504 leader_election.cc:304] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 253c339569f540d495975da5e18c19ce, 7535bab0b3954be89838f8ec2858c50e; no voters:
03:41:06.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.884321 27661 raft_consensus.cc:2802] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Leader pre-election won for term 1
03:41:06.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.884685 27661 raft_consensus.cc:491] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:41:06.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.885051 27661 raft_consensus.cc:3058] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 0 FOLLOWER]: Advancing to term 1
03:41:06.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.888973 27434 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d68cd3d0359945dcaa09e7c738153a22" candidate_uuid: "253c339569f540d495975da5e18c19ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" is_pre_election: true
03:41:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.889839 27434 raft_consensus.cc:2466] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 253c339569f540d495975da5e18c19ce in term 0.
03:41:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.895706 27661 raft_consensus.cc:513] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.898835 27434 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d68cd3d0359945dcaa09e7c738153a22" candidate_uuid: "7535bab0b3954be89838f8ec2858c50e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" is_pre_election: true
03:41:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.899325 27661 leader_election.cc:290] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [CANDIDATE]: Term 1 election: Requested vote from peers 7535bab0b3954be89838f8ec2858c50e (127.26.57.193:42417), 93d76ef06b6f4b02971cfe687ee8c75a (127.26.57.194:43383)
03:41:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.899459 27434 raft_consensus.cc:2466] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7535bab0b3954be89838f8ec2858c50e in term 0.
03:41:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.900703 27286 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d68cd3d0359945dcaa09e7c738153a22" candidate_uuid: "253c339569f540d495975da5e18c19ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7535bab0b3954be89838f8ec2858c50e"
03:41:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.900776 27210 leader_election.cc:304] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7535bab0b3954be89838f8ec2858c50e, 93d76ef06b6f4b02971cfe687ee8c75a; no voters:
03:41:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.901320 27286 raft_consensus.cc:3058] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 0 FOLLOWER]: Advancing to term 1
03:41:06.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.902592 27434 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d68cd3d0359945dcaa09e7c738153a22" candidate_uuid: "253c339569f540d495975da5e18c19ce" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93d76ef06b6f4b02971cfe687ee8c75a"
03:41:06.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.903151 27434 raft_consensus.cc:3058] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 0 FOLLOWER]: Advancing to term 1
03:41:06.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.906941 27582 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "d68cd3d0359945dcaa09e7c738153a22" candidate_uuid: "7535bab0b3954be89838f8ec2858c50e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "253c339569f540d495975da5e18c19ce" is_pre_election: true
03:41:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.907819 27582 raft_consensus.cc:2391] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 7535bab0b3954be89838f8ec2858c50e in current term 1: Already voted for candidate 253c339569f540d495975da5e18c19ce in this term.
03:41:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.907871 27286 raft_consensus.cc:2466] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 253c339569f540d495975da5e18c19ce in term 1.
03:41:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.908356 27659 raft_consensus.cc:2762] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
03:41:06.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.908998 27504 leader_election.cc:304] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 253c339569f540d495975da5e18c19ce, 7535bab0b3954be89838f8ec2858c50e; no voters:
03:41:06.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.909834 27661 raft_consensus.cc:2802] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 1 FOLLOWER]: Leader election won for term 1
03:41:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.911103 27434 raft_consensus.cc:2466] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 253c339569f540d495975da5e18c19ce in term 1.
03:41:06.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.912791 27661 raft_consensus.cc:695] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [term 1 LEADER]: Becoming Leader. State: Replica: 253c339569f540d495975da5e18c19ce, State: Running, Role: LEADER
03:41:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.913867 27661 consensus_queue.cc:237] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } }
03:41:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.923977 27687 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: TxnStatusTablet state changed. Reason: New leader 253c339569f540d495975da5e18c19ce. Latest consensus state: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.924772 27687 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: This TxnStatusTablet replica's current role is: LEADER
03:41:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.929579 27689 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
03:41:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.930172 27689 txn_status_manager.cc:930] Loading transaction status metadata into memory...
03:41:06.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.925115 27073 catalog_manager.cc:5582] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce reported cstate change: term changed from 0 to 1, leader changed from <none> to 253c339569f540d495975da5e18c19ce (127.26.57.195). New cstate: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } health_report { overall_health: HEALTHY } } }
03:41:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:06.968046 27660 consensus_queue.cc:1035] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [LEADER]: Connected to new peer: Peer: permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:41:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.003029 27660 consensus_queue.cc:1035] T 73be950dd4ec4cc2ad8aef3302ae5e80 P 93d76ef06b6f4b02971cfe687ee8c75a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:41:07.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.027160 27434 raft_consensus.cc:1273] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a [term 1 FOLLOWER]: Refusing update from remote peer 253c339569f540d495975da5e18c19ce: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:41:07.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.029060 27285 raft_consensus.cc:1273] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e [term 1 FOLLOWER]: Refusing update from remote peer 253c339569f540d495975da5e18c19ce: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:41:07.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.028831 27661 consensus_queue.cc:1035] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [LEADER]: Connected to new peer: Peer: permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:41:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.032006 27661 consensus_queue.cc:1035] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce [LEADER]: Connected to new peer: Peer: permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:41:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.060060 27659 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: TxnStatusTablet state changed. Reason: New leader 253c339569f540d495975da5e18c19ce. Latest consensus state: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.060757 27659 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: This TxnStatusTablet replica's current role is: FOLLOWER
03:41:07.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.070781 27660 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: TxnStatusTablet state changed. Reason: New leader 253c339569f540d495975da5e18c19ce. Latest consensus state: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:07.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.071470 27660 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: This TxnStatusTablet replica's current role is: FOLLOWER
03:41:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.079852 27660 consensus_queue.cc:1035] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:41:07.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.089952 27695 mvcc.cc:204] Tried to move back new op lower bound from 7171370463326584832 to 7171370462887387136. Current Snapshot: MvccSnapshot[applied={T|T < 7171370463326584832}]
03:41:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.094229 27693 mvcc.cc:204] Tried to move back new op lower bound from 7171370463326584832 to 7171370462887387136. Current Snapshot: MvccSnapshot[applied={T|T < 7171370463326584832}]
03:41:07.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.111891 27698 mvcc.cc:204] Tried to move back new op lower bound from 7171370463326584832 to 7171370462887387136. Current Snapshot: MvccSnapshot[applied={T|T < 7171370463326584832}]
03:41:07.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.133841 27694 consensus_queue.cc:1035] T a28195f20b844748a793f5c96e9fd3fd P 93d76ef06b6f4b02971cfe687ee8c75a [LEADER]: Connected to new peer: Peer: permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:41:07.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.136897 27687 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:07.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.142627 27687 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: This TxnStatusTablet replica's current role is: LEADER
03:41:07.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.144862 27667 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.162803 27659 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.163522 27659 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 7535bab0b3954be89838f8ec2858c50e: This TxnStatusTablet replica's current role is: FOLLOWER
03:41:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.183313 27667 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 93d76ef06b6f4b02971cfe687ee8c75a: This TxnStatusTablet replica's current role is: FOLLOWER
03:41:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.203692 27687 tablet_replica.cc:440] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "253c339569f540d495975da5e18c19ce" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7535bab0b3954be89838f8ec2858c50e" member_type: VOTER last_known_addr { host: "127.26.57.193" port: 42417 } } peers { permanent_uuid: "93d76ef06b6f4b02971cfe687ee8c75a" member_type: VOTER last_known_addr { host: "127.26.57.194" port: 43383 } } peers { permanent_uuid: "253c339569f540d495975da5e18c19ce" member_type: VOTER last_known_addr { host: "127.26.57.195" port: 45185 } } }
03:41:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.204432 27687 tablet_replica.cc:443] T d68cd3d0359945dcaa09e7c738153a22 P 253c339569f540d495975da5e18c19ce: This TxnStatusTablet replica's current role is: LEADER
03:41:07.235 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
03:41:07.242 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 97444f6fe8d648a5896ea26c78b964d0
03:41:07.246 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 73be950dd4ec4cc2ad8aef3302ae5e80 for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
03:41:07.247 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet a28195f20b844748a793f5c96e9fd3fd for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
03:41:07.247 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=73be950dd4ec4cc2ad8aef3302ae5e80@[253c339569f540d495975da5e18c19ce(127.26.57.195:45185),7535bab0b3954be89838f8ec2858c50e(127.26.57.193:42417),93d76ef06b6f4b02971cfe687ee8c75a(127.26.57.194:43383)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=a28195f20b844748a793f5c96e9fd3fd@[253c339569f540d495975da5e18c19ce(127.26.57.195:45185),7535bab0b3954be89838f8ec2858c50e(127.26.57.193:42417),93d76ef06b6f4b02971cfe687ee8c75a(127.26.57.194:43383)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
03:41:07.247 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC GetTableLocations: sending RPC to server master-127.26.57.252:35693, [4ms] Sub RPC GetTableLocations: received response from server master-127.26.57.252:35693: OK, deferred=null) after lookup
03:41:07.248 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 93d76ef06b6f4b02971cfe687ee8c75a(127.26.57.194:43383)] connecting to peer
03:41:07.248 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 93d76ef06b6f4b02971cfe687ee8c75a(127.26.57.194:43383)] Successfully connected to peer
03:41:07.285 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x982c4823, L:/127.0.0.1:42542 - R:/127.26.57.194:43383] using TOKEN/null
03:41:07.394 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.57.252:35693
03:41:07.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:41:07.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.395643 26855 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu with pid 27032
03:41:07.430 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.57.252:35693(127.26.57.252:35693)] handling channelInactive
03:41:07.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state READY due to: connection closed
03:41:07.433 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:41:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:07.542857 27479 heartbeater.cc:646] Failed to heartbeat to 127.26.57.252:35693 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.633803 27714 raft_consensus.cc:491] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a8af068477b047b4b94dd6f9f3a5dfa0)
03:41:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.634626 27715 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 1 FOLLOWER]: Starting pre-election (detected failure of leader a8af068477b047b4b94dd6f9f3a5dfa0)
03:41:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.634711 27714 raft_consensus.cc:513] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:41:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.635258 27715 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:41:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.637562 27715 leader_election.cc:290] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b4f379c72e104c409a5a2e6694778a9e (127.26.57.253:39363), a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693)
03:41:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.637820 26909 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b4f379c72e104c409a5a2e6694778a9e" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4fe4841fc4024d5ba14f2def054c512d" is_pre_election: true
03:41:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.638584 26909 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4f379c72e104c409a5a2e6694778a9e in term 1.
03:41:07.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.639981 26995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fe4841fc4024d5ba14f2def054c512d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b4f379c72e104c409a5a2e6694778a9e" is_pre_election: true
03:41:07.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.640859 26995 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4fe4841fc4024d5ba14f2def054c512d in term 1.
03:41:07.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.642397 26884 leader_election.cc:304] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4fe4841fc4024d5ba14f2def054c512d, b4f379c72e104c409a5a2e6694778a9e; no voters:
03:41:07.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.643105 26971 leader_election.cc:304] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4fe4841fc4024d5ba14f2def054c512d, b4f379c72e104c409a5a2e6694778a9e; no voters:
03:41:07.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:07.644011 26885 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111) [suppressed 13 similar messages]
03:41:07.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.646165 27715 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 1 FOLLOWER]: Leader pre-election won for term 2
03:41:07.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.646497 27715 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 1 FOLLOWER]: Starting leader election (detected failure of leader a8af068477b047b4b94dd6f9f3a5dfa0)
03:41:07.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.646791 27715 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 1 FOLLOWER]: Advancing to term 2
03:41:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.642335 27714 leader_election.cc:290] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 4fe4841fc4024d5ba14f2def054c512d (127.26.57.254:45045), a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693)
03:41:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.650594 27714 raft_consensus.cc:2802] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 1 FOLLOWER]: Leader pre-election won for term 2
03:41:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.651744 27714 raft_consensus.cc:491] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 1 FOLLOWER]: Starting leader election (detected failure of leader a8af068477b047b4b94dd6f9f3a5dfa0)
03:41:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.652626 27714 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 1 FOLLOWER]: Advancing to term 2
03:41:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:07.653681 26885 leader_election.cc:336] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:07.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:07.655293 26970 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111) [suppressed 6 similar messages]
03:41:07.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.656719 27715 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:41:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.659526 26995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fe4841fc4024d5ba14f2def054c512d" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b4f379c72e104c409a5a2e6694778a9e"
03:41:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.663019 27715 leader_election.cc:290] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 2 election: Requested vote from peers b4f379c72e104c409a5a2e6694778a9e (127.26.57.253:39363), a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693)
03:41:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:07.665064 26885 leader_election.cc:336] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.668167 27714 raft_consensus.cc:513] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:41:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.669289 26995 raft_consensus.cc:2391] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 4fe4841fc4024d5ba14f2def054c512d in current term 2: Already voted for candidate b4f379c72e104c409a5a2e6694778a9e in this term.
03:41:07.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:07.669943 26970 leader_election.cc:336] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.671074 27714 leader_election.cc:290] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [CANDIDATE]: Term 2 election: Requested vote from peers 4fe4841fc4024d5ba14f2def054c512d (127.26.57.254:45045), a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693)
03:41:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.672941 26909 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b4f379c72e104c409a5a2e6694778a9e" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4fe4841fc4024d5ba14f2def054c512d"
03:41:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.673818 26909 raft_consensus.cc:2391] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate b4f379c72e104c409a5a2e6694778a9e in current term 2: Already voted for candidate 4fe4841fc4024d5ba14f2def054c512d in this term.
03:41:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.675254 26884 leader_election.cc:304] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4fe4841fc4024d5ba14f2def054c512d; no voters: a8af068477b047b4b94dd6f9f3a5dfa0, b4f379c72e104c409a5a2e6694778a9e
03:41:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.676059 27715 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:41:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:07.676555 26970 leader_election.cc:336] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.676887 26970 leader_election.cc:304] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b4f379c72e104c409a5a2e6694778a9e; no voters: 4fe4841fc4024d5ba14f2def054c512d, a8af068477b047b4b94dd6f9f3a5dfa0
03:41:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.677454 27714 raft_consensus.cc:2747] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
03:41:07.749 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.751 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.26.57.252:35693(127.26.57.252:35693) for tablet Kudu Master: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.769 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.771 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.771 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.777 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:07.789 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.793 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.793 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.794 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.796 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:07.809 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.811 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:07.829 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.831 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.831 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.831 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.833 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:07.869 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.870 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.870 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.870 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.872 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:07.909 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.914 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.917 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.917 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.917 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:07.969 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:07.970 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.970 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:07.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.997865 27715 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:41:07.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.998351 27715 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:41:08.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:07.999981 27715 leader_election.cc:290] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b4f379c72e104c409a5a2e6694778a9e (127.26.57.253:39363), a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693)
03:41:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.000844 26995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fe4841fc4024d5ba14f2def054c512d" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b4f379c72e104c409a5a2e6694778a9e" is_pre_election: true
03:41:08.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.001420 26995 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4fe4841fc4024d5ba14f2def054c512d in term 2.
03:41:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.002282 26884 leader_election.cc:304] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4fe4841fc4024d5ba14f2def054c512d, b4f379c72e104c409a5a2e6694778a9e; no voters:
03:41:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.002990 27715 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Leader pre-election won for term 3
03:41:08.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.003340 27715 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:41:08.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.004329 26885 leader_election.cc:336] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:08.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.005020 27715 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 2 FOLLOWER]: Advancing to term 3
03:41:08.010 [DEBUG - pool-38-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] sending RPC to server master-127.26.57.252:35693, [2ms] received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [22ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [25ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [28ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [28ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [39ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [43ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [44ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [47ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [47ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [62ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [63ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [82ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [83ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [121ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [123ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [123ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [159ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [164ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [165ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [168ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [168ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [221ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [223ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [223ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, deferred=Deferred@212294182(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1395426687@7c21c8ce -> (continuation of Deferred@2078809490 after retry RPC after error@1350654425) -> (continuation of Deferred@1723944837 after retry RPC after error@349477931) -> (continuation of Deferred@795547566 after retry RPC after error@1971248139) -> (continuation of Deferred@430905414 after retry RPC after error@690682524) -> (continuation of Deferred@1577799280 after retry RPC after error@1570131774) -> (continuation of Deferred@1008341474 after retry RPC after error@1117450983) -> (continuation of Deferred@1181109258 after retry RPC after error@702940231), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/1883816157@459f8598 -> (continuation of Deferred@2078809490 after retry RPC after error@1350654425) -> (continuation of Deferred@1723944837 after retry RPC after error@349477931) -> (continuation of Deferred@795547566 after retry RPC after error@1971248139) -> (continuation of Deferred@430905414 after retry RPC after error@690682524) -> (continuation of Deferred@1577799280 after retry RPC after error@1570131774) -> (continuation of Deferred@1008341474 after retry RPC after error@1117450983) -> (continuation of Deferred@1181109258 after retry RPC after error@702940231)))
03:41:08.012 [WARN - pool-38-thread-1] (HashedWheelTimer.java:716) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501)
at java.lang.Thread.run(Thread.java:748)
03:41:08.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.012324 27715 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:41:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.014335 27715 leader_election.cc:290] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 3 election: Requested vote from peers b4f379c72e104c409a5a2e6694778a9e (127.26.57.253:39363), a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693)
03:41:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.015209 26995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4fe4841fc4024d5ba14f2def054c512d" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "b4f379c72e104c409a5a2e6694778a9e"
03:41:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.015733 26995 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 2 FOLLOWER]: Advancing to term 3
03:41:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.018865 26885 leader_election.cc:336] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.022747 26995 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4fe4841fc4024d5ba14f2def054c512d in term 3.
03:41:08.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.023854 26884 leader_election.cc:304] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4fe4841fc4024d5ba14f2def054c512d, b4f379c72e104c409a5a2e6694778a9e; no voters: a8af068477b047b4b94dd6f9f3a5dfa0
03:41:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.024662 27715 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 3 FOLLOWER]: Leader election won for term 3
03:41:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.025729 27715 raft_consensus.cc:695] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [term 3 LEADER]: Becoming Leader. State: Replica: 4fe4841fc4024d5ba14f2def054c512d, State: Running, Role: LEADER
03:41:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.026815 27715 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } }
03:41:08.029 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:08.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.033025 27721 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4fe4841fc4024d5ba14f2def054c512d. Latest consensus state: current_term: 3 leader_uuid: "4fe4841fc4024d5ba14f2def054c512d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:41:08.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.033865 27721 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: This master's current role is: LEADER
03:41:08.034 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:08.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.035931 27723 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
03:41:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.043038 27723 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0]
03:41:08.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.044328 27723 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=d48c17c316d549cc9828d9b1ace7f686]
03:41:08.049 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:08.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.051 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.049275 27723 tablet_loader.cc:96] loaded metadata for tablet 73be950dd4ec4cc2ad8aef3302ae5e80 (table txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0])
03:41:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.050757 27723 tablet_loader.cc:96] loaded metadata for tablet a28195f20b844748a793f5c96e9fd3fd (table txn_manager_fallback [id=97444f6fe8d648a5896ea26c78b964d0])
03:41:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.052029 27723 tablet_loader.cc:96] loaded metadata for tablet d68cd3d0359945dcaa09e7c738153a22 (table kudu_system.kudu_transactions [id=d48c17c316d549cc9828d9b1ace7f686])
03:41:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.053187 27723 catalog_manager.cc:1486] Initializing Kudu cluster ID...
03:41:08.054 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045; will retry
03:41:08.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.056466 27723 catalog_manager.cc:1261] Loaded cluster ID: ae79a737d3d24e1a8d8afdc6d0b7ae66
03:41:08.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.056700 27723 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
03:41:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.060055 27723 catalog_manager.cc:1506] Loading token signing keys...
03:41:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.063088 27723 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d: Loaded TSK: 0
03:41:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.064143 27723 catalog_manager.cc:1516] Initializing in-progress tserver states...
03:41:08.069 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.26.57.252:35693(127.26.57.252:35693)] connecting to peer
03:41:08.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.252:35693(127.26.57.252:35693)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.57.252:35693: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693
03:41:08.074 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:41:08.074 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=3, TimeoutTracker(timeout=250, elapsed=45), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [2ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [3ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [23ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [24ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [26ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [43ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [45ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, deferred=Deferred@726939113(state=PENDING, result=null, callback=(continuation of Deferred@737434771 after retry RPC after error@277997274) -> (continuation of Deferred@649148425 after retry RPC after error@742435403), errback=(continuation of Deferred@737434771 after retry RPC after error@277997274) -> (continuation of Deferred@649148425 after retry RPC after error@742435403))) after lookup
03:41:08.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.127110 26995 raft_consensus.cc:1273] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [term 3 FOLLOWER]: Refusing update from remote peer 4fe4841fc4024d5ba14f2def054c512d: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
03:41:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.129666 27721 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.001s
03:41:08.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.134835 26885 consensus_peers.cc:487] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d -> Peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Couldn't send request to peer a8af068477b047b4b94dd6f9f3a5dfa0. Status: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:41:08.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.151374 27714 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4fe4841fc4024d5ba14f2def054c512d. Latest consensus state: current_term: 3 leader_uuid: "4fe4841fc4024d5ba14f2def054c512d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:41:08.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.152189 27714 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: This master's current role is: FOLLOWER
03:41:08.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.214037 27331 heartbeater.cc:646] Failed to heartbeat to 127.26.57.252:35693 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:08.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.278925 27714 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4fe4841fc4024d5ba14f2def054c512d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:41:08.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.279659 27714 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f379c72e104c409a5a2e6694778a9e [sys.catalog]: This master's current role is: FOLLOWER
03:41:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.280725 26930 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:42168 (request call id 11) took 205 ms (client timeout 205 ms). Trace:
03:41:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.281175 26930 rpcz_store.cc:269] 0625 03:41:08.075267 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:41:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:41:08.075450 (+ 183us) service_pool.cc:225] Handling call
03:41:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 03:41:08.280489 (+205039us) inbound_call.cc:173] Queueing success response
03:41:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":665,"init.queue_time_us":2481,"thread_start_us":66908,"threads_started":5}
03:41:08.284 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=4, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [2ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [3ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [23ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [24ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [26ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [43ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [45ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [46ms] sending RPC to server master-127.26.57.254:45045, [255ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@726939113(state=PENDING, result=null, callback=(continuation of Deferred@737434771 after retry RPC after error@277997274) -> (continuation of Deferred@649148425 after retry RPC after error@742435403) -> (continuation of Deferred@635005844 after retry RPC@979500739), errback=(continuation of Deferred@737434771 after retry RPC after error@277997274) -> (continuation of Deferred@649148425 after retry RPC after error@742435403) -> (continuation of Deferred@635005844 after retry RPC@979500739)))
03:41:08.284 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=4, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [2ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [3ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [23ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [24ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [26ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [26ms] delaying RPC due to: Service unavailable: Master config (127.26.57.252:35693,127.26.57.253:39363,127.26.57.254:45045) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.252:35693, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.253:39363, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.57.254:45045, [43ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.252:35693: Network error: Failed to connect to peer master-127.26.57.252:35693(127.26.57.252:35693): Connection refused: /127.26.57.252:35693, [45ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.253:39363: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.26.57.254:45045: OK, [46ms] sending RPC to server master-127.26.57.254:45045, [255ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@726939113(state=PENDING, result=null, callback=(continuation of Deferred@737434771 after retry RPC after error@277997274) -> (continuation of Deferred@649148425 after retry RPC after error@742435403) -> (continuation of Deferred@635005844 after retry RPC@979500739), errback=(continuation of Deferred@737434771 after retry RPC after error@277997274) -> (continuation of Deferred@649148425 after retry RPC after error@742435403) -> (continuation of Deferred@635005844 after retry RPC@979500739)))
03:41:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.300148 27715 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4fe4841fc4024d5ba14f2def054c512d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4fe4841fc4024d5ba14f2def054c512d" member_type: VOTER last_known_addr { host: "127.26.57.254" port: 45045 } } peers { permanent_uuid: "b4f379c72e104c409a5a2e6694778a9e" member_type: VOTER last_known_addr { host: "127.26.57.253" port: 39363 } } peers { permanent_uuid: "a8af068477b047b4b94dd6f9f3a5dfa0" member_type: VOTER last_known_addr { host: "127.26.57.252" port: 35693 } } }
03:41:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.301496 27715 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d [sys.catalog]: This master's current role is: LEADER
03:41:08.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.314031 27627 heartbeater.cc:646] Failed to heartbeat to 127.26.57.252:35693 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111)
03:41:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.349056 27511 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.004s from last keepalive heartbeat (effective timeout is 1.000s)
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:08.332460 26898 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:42182:
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
03:41:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
03:41:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
03:41:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
03:41:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
03:41:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
03:41:08.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.370479 27519 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.26.57.252:35693)
03:41:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.371212 26898 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
03:41:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.395008 27744 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
03:41:08.410 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 253c339569f540d495975da5e18c19ce (127.26.57.195:45185): transaction ID 0 is not available for further commits: state: ABORT_IN_PROGRESS user: "slave" start_timestamp: 1750822867 last_transition_timestamp: 1750822868
03:41:08.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:08.680811 26885 consensus_peers.cc:487] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d -> Peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Couldn't send request to peer a8af068477b047b4b94dd6f9f3a5dfa0. Status: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:41:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:09.201002 26885 consensus_peers.cc:487] T 00000000000000000000000000000000 P 4fe4841fc4024d5ba14f2def054c512d -> Peer a8af068477b047b4b94dd6f9f3a5dfa0 (127.26.57.252:35693): Couldn't send request to peer a8af068477b047b4b94dd6f9f3a5dfa0. Status: Network error: Client connection negotiation failed: client connection to 127.26.57.252:35693: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
03:41:09.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:09.258731 27335 heartbeater.cc:499] Master 127.26.57.254:45045 was elected leader, sending a full tablet report...
03:41:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:09.308535 27629 heartbeater.cc:499] Master 127.26.57.254:45045 was elected leader, sending a full tablet report...
03:41:09.441 [DEBUG - main] (Connection.java:575) [peer 93d76ef06b6f4b02971cfe687ee8c75a(127.26.57.194:43383)] disconnecting while in state READY
03:41:09.441 [DEBUG - main] (Connection.java:575) [peer master-127.26.57.252:35693(127.26.57.252:35693)] disconnecting while in state TERMINATED
03:41:09.441 [DEBUG - main] (Connection.java:575) [peer master-127.26.57.254:45045(127.26.57.254:45045)] disconnecting while in state READY
03:41:09.441 [DEBUG - main] (Connection.java:575) [peer master-127.26.57.253:39363(127.26.57.253:39363)] disconnecting while in state READY
03:41:09.442 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.57.254:45045(127.26.57.254:45045)] handling channelInactive
03:41:09.442 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.57.254:45045(127.26.57.254:45045)] cleaning up while in state READY due to: connection closed
03:41:09.442 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:41:09.443 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 93d76ef06b6f4b02971cfe687ee8c75a(127.26.57.194:43383)] handling channelInactive
03:41:09.443 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 93d76ef06b6f4b02971cfe687ee8c75a(127.26.57.194:43383)] cleaning up while in state READY due to: connection closed
03:41:09.443 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.57.253:39363(127.26.57.253:39363)] handling channelInactive
03:41:09.443 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.57.253:39363(127.26.57.253:39363)] cleaning up while in state READY due to: connection closed
03:41:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:09.443358 26855 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu with pid 27182
03:41:09.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:09.471802 26855 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu with pid 27332
03:41:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:09.504781 27506 connection.cc:537] client connection to 127.26.57.194:43383 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:41:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 03:41:09.505271 27506 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:41:09.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:09.505977 26855 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu with pid 27484
03:41:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:09.542659 26855 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu with pid 26867
03:41:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 03:41:09.577986 26855 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task3o63Jh/build/tsan/bin/kudu with pid 26947
03:41:10.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T03:41:10Z chronyd exiting
03:41:11.794 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 253c339569f540d495975da5e18c19ce (127.26.57.195:45185): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1750822867 last_transition_timestamp: 1750822868
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 253c339569f540d495975da5e18c19ce (127.26.57.195:45185): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1750822867 last_transition_timestamp: 1750822868
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more