09:14:05.887 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:14:05.888 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8A6J2s/test-tmp
09:14:05.888 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:14:05.888 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8A6J2s/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:14:05.889 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:14:06.214 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:14:06.214 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:14:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.217223 30238 mini_oidc.cc:150] Starting JWKS server
09:14:06.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.220600 30238 webserver.cc:492] Webserver started at http://127.0.0.1:38973/ using document root <none> and password file <none>
09:14:06.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.221009 30238 mini_oidc.cc:186] Starting OIDC Discovery server
09:14:06.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.222605 30238 webserver.cc:492] Webserver started at http://127.0.0.1:38857/ using document root <none> and password file <none>
09:14:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T09:14:06Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:14:06.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T09:14:06Z Disabled control of system clock
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.283179 30238 external_mini_cluster.cc:1366] Running /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/wal
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/logs
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data/info.pb
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:06.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.190:41627
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.190
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38973/jwks.json
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:14:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:14:06.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.579099 30250 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:06.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.579710 30250 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:06.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.580181 30250 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:06.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.611632 30250 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:14:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.611918 30250 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:14:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.612071 30250 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.612262 30250 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:14:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.612452 30250 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:14:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.612661 30250 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:14:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.612804 30250 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38973/jwks.json
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.648296 30250 master_runner.cc:386] Master server non-default flags:
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/wal
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.190:41627
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38973/jwks.json
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data/info.pb
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.190
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/logs
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:20 UTC on 5fd53c4cbb9d
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8773
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.649583 30250 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:06.651149 30250 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:06.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.663095 30256 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:06.663309 30257 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:07.834672 30259 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:07.838758 30258 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1176 milliseconds
09:14:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:07.839691 30250 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.178s user 0.398s sys 0.772s
09:14:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:07.840052 30250 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.178s user 0.398s sys 0.772s
09:14:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.840356 30250 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.842029 30250 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.845304 30250 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:07.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.846887 30250 hybrid_clock.cc:648] HybridClock initialized: now 1761729247846847 us; error 35 us; skew 500 ppm
09:14:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.848040 30250 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.856886 30250 webserver.cc:492] Webserver started at http://127.29.135.190:33205/ using document root <none> and password file <none>
09:14:07.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.858347 30250 fs_manager.cc:362] Metadata directory not provided
09:14:07.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.858657 30250 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.859320 30250 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:14:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.865892 30250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data/instance:
09:14:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f1ff44df6544f6ab375ae8c1fa999f6"
09:14:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:07 on dist-test-slave-td26"
09:14:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.867362 30250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/wal/instance:
09:14:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f1ff44df6544f6ab375ae8c1fa999f6"
09:14:07.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:07 on dist-test-slave-td26"
09:14:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.875628 30250 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
09:14:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.881412 30266 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.882537 30250 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.882791 30250 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data,/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/wal
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f1ff44df6544f6ab375ae8c1fa999f6"
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:07 on dist-test-slave-td26"
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.883050 30250 fs_report.cc:389] FS layout report
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/wal
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/wal
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data/data
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:07.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.953352 30250 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.954842 30250 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:07.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:07.955217 30250 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:08.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.034852 30250 rpc_server.cc:307] RPC server started. Bound to: 127.29.135.190:41627
09:14:08.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.034919 30327 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.135.190:41627 every 8 connection(s)
09:14:08.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.037601 30250 server_base.cc:1179] Dumped server information to /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/data/info.pb
09:14:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.043365 30328 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:08.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.047878 30238 external_mini_cluster.cc:1428] Started /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu as pid 30250
09:14:08.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.048317 30238 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-0/wal/instance
09:14:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.051640 30238 external_mini_cluster.cc:1366] Running /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/wal
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/logs
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data/info.pb
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.189:35713
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.189
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:08.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38973/jwks.json
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:14:08.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.053603 30328 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } has no permanent_uuid. Determining permanent_uuid...
09:14:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.080318 30328 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } has no permanent_uuid. Determining permanent_uuid...
09:14:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.082568 30269 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:08.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.087314 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.189:35713: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:08.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.136953 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } attempt: 1
09:14:08.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.141690 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.189:35713: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.210072 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } attempt: 2
09:14:08.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.214720 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.189:35713: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.306157 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } attempt: 3
09:14:08.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.310568 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.189:35713: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:08.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.365864 30330 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:08.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.366513 30330 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.367071 30330 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.398767 30330 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:14:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.399124 30330 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:14:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.399411 30330 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.399682 30330 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:14:08.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.399967 30330 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:14:08.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.400223 30330 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:14:08.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.400436 30330 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38973/jwks.json
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.435777 30330 master_runner.cc:386] Master server non-default flags:
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/wal
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.189:35713
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:08.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38973/jwks.json
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data/info.pb
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.189
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/logs
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:20 UTC on 5fd53c4cbb9d
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8773
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.437386 30330 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:08.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.438984 30330 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:08.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.450268 30339 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.453955 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } attempt: 4
09:14:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.460932 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.189:35713: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:08.760476 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } attempt: 5
09:14:08.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.767701 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.189:35713: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:09.315286 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } attempt: 6
09:14:09.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:09.339866 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.189:35713: Network error: Client connection negotiation failed: client connection to 127.29.135.189:35713: connect: Connection refused (error 111)
09:14:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:09.853624 30338 debug-util.cc:398] Leaking SignalData structure 0x7b0800037940 after lost signal to thread 30330
09:14:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.013427 30330 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.563s user 0.571s sys 0.985s
09:14:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.013814 30330 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.564s user 0.571s sys 0.985s
09:14:10.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:08.450974 30340 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:10.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.015780 30342 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.018625 30341 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1564 milliseconds
09:14:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.018697 30330 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:10.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.019804 30330 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:10.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.022192 30330 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.023521 30330 hybrid_clock.cc:648] HybridClock initialized: now 1761729250023475 us; error 55 us; skew 500 ppm
09:14:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.024303 30330 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:10.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.030484 30330 webserver.cc:492] Webserver started at http://127.29.135.189:44509/ using document root <none> and password file <none>
09:14:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.031394 30330 fs_manager.cc:362] Metadata directory not provided
09:14:10.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.031581 30330 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.031963 30330 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:14:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.036135 30330 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data/instance:
09:14:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "317bacb8d32f45ee92c35da517f50844"
09:14:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:10 on dist-test-slave-td26"
09:14:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.037227 30330 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/wal/instance:
09:14:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "317bacb8d32f45ee92c35da517f50844"
09:14:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:10 on dist-test-slave-td26"
09:14:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.044169 30330 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
09:14:10.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.049666 30351 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:10.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.050685 30330 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.050971 30330 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data,/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/wal
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "317bacb8d32f45ee92c35da517f50844"
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:10 on dist-test-slave-td26"
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.051254 30330 fs_report.cc:389] FS layout report
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/wal
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/wal
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data/data
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.097406 30330 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.098824 30330 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.099229 30330 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:10.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.184202 30330 rpc_server.cc:307] RPC server started. Bound to: 127.29.135.189:35713
09:14:10.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.184315 30412 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.135.189:35713 every 8 connection(s)
09:14:10.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.186975 30330 server_base.cc:1179] Dumped server information to /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/data/info.pb
09:14:10.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.192544 30413 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.195704 30238 external_mini_cluster.cc:1428] Started /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu as pid 30330
09:14:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.196166 30238 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-1/wal/instance
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.199196 30238 external_mini_cluster.cc:1366] Running /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/wal
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/logs
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data/info.pb
09:14:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.188:44461
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.188
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38973/jwks.json
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
09:14:10.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
09:14:10.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.208335 30413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } has no permanent_uuid. Determining permanent_uuid...
09:14:10.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.230744 30413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } has no permanent_uuid. Determining permanent_uuid...
09:14:10.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.245283 30413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } has no permanent_uuid. Determining permanent_uuid...
09:14:10.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.247836 30352 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.251889 30413 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.301357 30413 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 1
09:14:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.305562 30413 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.373932 30413 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 2
09:14:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.377907 30413 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.400537 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } attempt: 7
09:14:10.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.411793 30328 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } has no permanent_uuid. Determining permanent_uuid...
09:14:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.417419 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.469292 30413 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 3
09:14:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.473088 30413 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.475803 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 1
09:14:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.479460 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.518457 30415 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.519106 30415 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:10.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.519716 30415 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.551614 30415 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:14:10.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.551954 30415 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
09:14:10.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.552204 30415 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:10.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.552520 30415 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:14:10.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.552800 30415 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:14:10.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.553146 30415 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:14:10.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.553396 30415 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38973/jwks.json
09:14:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.560914 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 2
09:14:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.564939 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.590801 30415 master_runner.cc:386] Master server non-default flags:
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/wal
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.188:44461
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38973/jwks.json
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data/info.pb
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.188
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/logs
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:20 UTC on 5fd53c4cbb9d
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8773
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.592435 30415 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.594048 30415 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:10.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.604574 30427 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.616540 30413 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 4
09:14:10.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.620358 30413 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.650329 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 3
09:14:10.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.655010 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.795394 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 4
09:14:10.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.799962 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:10.919791 30413 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 5
09:14:10.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.925655 30413 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:11.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.083369 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 5
09:14:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:11.087208 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.473071 30413 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 6
09:14:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:11.479653 30413 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:11.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.639865 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 6
09:14:11.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:11.646804 30328 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.135.188:44461: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.605871 30428 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:10.609468 30430 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:11.774394 30429 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
09:14:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.774513 30415 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.778431 30415 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.781466 30415 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.782912 30415 hybrid_clock.cc:648] HybridClock initialized: now 1761729251782869 us; error 58 us; skew 500 ppm
09:14:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.783795 30415 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.790452 30415 webserver.cc:492] Webserver started at http://127.29.135.188:38499/ using document root <none> and password file <none>
09:14:11.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.791313 30415 fs_manager.cc:362] Metadata directory not provided
09:14:11.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.791499 30415 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.791890 30415 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:14:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.796417 30415 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data/instance:
09:14:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f85076611264064a8c1d15996e30278"
09:14:11.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:11 on dist-test-slave-td26"
09:14:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.797544 30415 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/wal/instance:
09:14:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f85076611264064a8c1d15996e30278"
09:14:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:11 on dist-test-slave-td26"
09:14:11.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.804875 30415 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
09:14:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.810549 30441 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.811532 30415 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.811854 30415 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data,/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/wal
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f85076611264064a8c1d15996e30278"
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:11 on dist-test-slave-td26"
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.812170 30415 fs_report.cc:389] FS layout report
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/wal
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/wal
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data/data
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:11.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.860219 30415 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.861708 30415 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.862147 30415 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.942915 30415 rpc_server.cc:307] RPC server started. Bound to: 127.29.135.188:44461
09:14:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.942994 30502 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.135.188:44461 every 8 connection(s)
09:14:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.945709 30415 server_base.cc:1179] Dumped server information to /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/data/info.pb
09:14:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.946594 30238 external_mini_cluster.cc:1428] Started /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu as pid 30415
09:14:11.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.947117 30238 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/master-2/wal/instance
09:14:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.952481 30503 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.968907 30503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } has no permanent_uuid. Determining permanent_uuid...
09:14:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:11.996790 30503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } has no permanent_uuid. Determining permanent_uuid...
09:14:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.005973 30503 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } has no permanent_uuid. Determining permanent_uuid...
09:14:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.022109 30503 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278: Bootstrap starting.
09:14:12.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.027774 30503 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278: Neither blocks nor log segments found. Creating new log.
09:14:12.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.029735 30503 log.cc:826] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278: Log is configured to *not* fsync() on all Append() calls
09:14:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.034391 30503 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278: No bootstrap required, opened a new log
09:14:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.060585 30503 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.061393 30503 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.061684 30503 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7f85076611264064a8c1d15996e30278, State: Initialized, Role: FOLLOWER
09:14:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.062531 30503 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.064811 30512 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.065390 30512 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.066260 30503 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: configured and running, proceeding with master startup.
09:14:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.069816 30512 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:12.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.070406 30512 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.073918 30512 leader_election.cc:290] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f1ff44df6544f6ab375ae8c1fa999f6 (127.29.135.190:41627), 317bacb8d32f45ee92c35da517f50844 (127.29.135.189:35713)
09:14:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.099457 30523 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:14:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.100708 30523 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:14:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.116919 30447 tablet.cc:2378] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:14:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.091583 30415 master_runner.cc:186] Error getting master registration for 127.29.135.190:41627: OK, instance_id { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" instance_seqno: 1761729248016947 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:14:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.122196 30415 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:14:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.461743 30445 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.29.135.190:41627 timed out after 0.388s (SENT)
09:14:12.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.540241 30413 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 7
09:14:12.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.574100 30413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: Bootstrap starting.
09:14:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.581646 30413 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: Neither blocks nor log segments found. Creating new log.
09:14:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.584146 30413 log.cc:826] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: Log is configured to *not* fsync() on all Append() calls
09:14:12.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.598387 30413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: No bootstrap required, opened a new log
09:14:12.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.617391 30413 raft_consensus.cc:359] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.618073 30413 raft_consensus.cc:385] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:12.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.618376 30413 raft_consensus.cc:740] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 317bacb8d32f45ee92c35da517f50844, State: Initialized, Role: FOLLOWER
09:14:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.619210 30413 consensus_queue.cc:260] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.626753 30529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.627748 30529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.628777 30413 sys_catalog.cc:565] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: configured and running, proceeding with master startup.
09:14:12.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.628970 30378 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f85076611264064a8c1d15996e30278" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "317bacb8d32f45ee92c35da517f50844" is_pre_election: true
09:14:12.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.629853 30378 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f85076611264064a8c1d15996e30278 in term 0.
09:14:12.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.630406 30378 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34302 (request call id 1) took 555 ms (client timeout 388 ms). Trace:
09:14:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.630796 30378 rpcz_store.cc:269] 1029 09:14:12.075320 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:14:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:12.075783 (+ 463us) service_pool.cc:225] Handling call
09:14:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:12.630331 (+554548us) inbound_call.cc:173] Queueing success response
09:14:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:14:12.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.632717 30377 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f85076611264064a8c1d15996e30278" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "317bacb8d32f45ee92c35da517f50844" is_pre_election: true
09:14:12.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.633376 30377 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f85076611264064a8c1d15996e30278 in term 0.
09:14:12.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.634662 30444 leader_election.cc:304] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 317bacb8d32f45ee92c35da517f50844, 7f85076611264064a8c1d15996e30278; no voters:
09:14:12.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.638211 30533 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:14:12.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.638526 30533 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:14:12.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.638789 30533 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 0 FOLLOWER]: Advancing to term 1
09:14:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.643110 30533 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.646724 30533 leader_election.cc:290] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [CANDIDATE]: Term 1 election: Requested vote from peers 4f1ff44df6544f6ab375ae8c1fa999f6 (127.29.135.190:41627), 317bacb8d32f45ee92c35da517f50844 (127.29.135.189:35713)
09:14:12.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.647641 30377 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f85076611264064a8c1d15996e30278" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "317bacb8d32f45ee92c35da517f50844"
09:14:12.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.648175 30377 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 0 FOLLOWER]: Advancing to term 1
09:14:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.656107 30377 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7f85076611264064a8c1d15996e30278 in term 1.
09:14:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.657411 30444 leader_election.cc:304] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 317bacb8d32f45ee92c35da517f50844, 7f85076611264064a8c1d15996e30278; no voters:
09:14:12.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.658219 30533 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 1 FOLLOWER]: Leader election won for term 1
09:14:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.661520 30533 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [term 1 LEADER]: Becoming Leader. State: Replica: 7f85076611264064a8c1d15996e30278, State: Running, Role: LEADER
09:14:12.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.662962 30533 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.677810 30538 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f85076611264064a8c1d15996e30278. Latest consensus state: current_term: 1 leader_uuid: "7f85076611264064a8c1d15996e30278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.678704 30538 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: This master's current role is: LEADER
09:14:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.680276 30328 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } attempt: 7
09:14:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.686041 30544 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:14:12.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.712646 30543 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:14:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.713030 30543 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:14:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.714545 30544 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:14:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.685288 30330 master_runner.cc:186] Error getting master registration for 127.29.135.190:41627: OK, instance_id { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" instance_seqno: 1761729248016947 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:14:12.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.739122 30330 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:14:12.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.741356 30328 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: Bootstrap starting.
09:14:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.751343 30377 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Refusing update from remote peer 7f85076611264064a8c1d15996e30278: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:14:12.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.753345 30538 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [LEADER]: Connected to new peer: Peer: permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:12.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.762526 30328 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: Neither blocks nor log segments found. Creating new log.
09:14:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.765590 30328 log.cc:826] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: Log is configured to *not* fsync() on all Append() calls
09:14:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.779569 30328 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: No bootstrap required, opened a new log
09:14:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.826836 30529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f85076611264064a8c1d15996e30278. Latest consensus state: current_term: 1 leader_uuid: "7f85076611264064a8c1d15996e30278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.830754 30328 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.831648 30328 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:12.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.831947 30328 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4f1ff44df6544f6ab375ae8c1fa999f6, State: Initialized, Role: FOLLOWER
09:14:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.833027 30529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.832986 30328 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.835218 30551 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.836094 30551 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.837421 30328 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: configured and running, proceeding with master startup.
09:14:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.837553 30293 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f85076611264064a8c1d15996e30278" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" is_pre_election: true
09:14:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.838426 30293 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7f85076611264064a8c1d15996e30278 in term 0.
09:14:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.838984 30293 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42602 (request call id 1) took 765 ms (client timeout 388 ms). Trace:
09:14:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.839315 30293 rpcz_store.cc:269] 1029 09:14:12.073339 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:14:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:12.073535 (+ 196us) service_pool.cc:225] Handling call
09:14:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:12.838905 (+765370us) inbound_call.cc:173] Queueing success response
09:14:12.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:14:12.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.841173 30291 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f85076611264064a8c1d15996e30278" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6"
09:14:12.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.841603 30290 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 0 FOLLOWER]: Advancing to term 1
09:14:12.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.849161 30292 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7f85076611264064a8c1d15996e30278" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" is_pre_election: true
09:14:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.850380 30292 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42602 (request call id 3) took 384 ms (client timeout 388 ms). Trace:
09:14:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.850822 30292 rpcz_store.cc:269] 1029 09:14:12.465531 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:14:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:12.465658 (+ 127us) service_pool.cc:225] Handling call
09:14:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:12.850311 (+384653us) inbound_call.cc:173] Queueing success response
09:14:12.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
09:14:12.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.854611 30290 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 1 FOLLOWER]: Refusing update from remote peer 7f85076611264064a8c1d15996e30278: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:14:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:12.856683 30357 tablet.cc:2378] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:14:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.856571 30538 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:14:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.890242 30551 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7f85076611264064a8c1d15996e30278. Latest consensus state: current_term: 1 leader_uuid: "7f85076611264064a8c1d15996e30278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.890955 30551 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.892298 30529 sys_catalog.cc:455] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7f85076611264064a8c1d15996e30278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.895789 30529 sys_catalog.cc:458] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.899308 30538 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7f85076611264064a8c1d15996e30278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.899951 30538 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: This master's current role is: LEADER
09:14:12.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.900496 30538 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7f85076611264064a8c1d15996e30278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.901141 30538 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278 [sys.catalog]: This master's current role is: LEADER
09:14:12.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.909869 30551 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7f85076611264064a8c1d15996e30278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.910715 30551 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: This master's current role is: FOLLOWER
09:14:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.917732 30544 catalog_manager.cc:1357] Generated new cluster ID: b4bbe5694ba54b4bb19b829763a5a65f
09:14:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.918035 30544 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:14:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.972502 30544 catalog_manager.cc:1380] Generated new certificate authority record
09:14:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:12.974726 30544 catalog_manager.cc:1514] Loading token signing keys...
09:14:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.012648 30544 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 7f85076611264064a8c1d15996e30278: Generated new TSK 0
09:14:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.014052 30544 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:14:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.014644 30568 catalog_manager.cc:1269] Loaded cluster ID: b4bbe5694ba54b4bb19b829763a5a65f
09:14:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.015048 30568 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: loading cluster ID for follower catalog manager: success
09:14:13.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.023931 30568 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: acquiring CA information for follower catalog manager: success
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.042152 30238 external_mini_cluster.cc:1366] Running /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/wal
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/logs
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data/info.pb
09:14:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.129:0
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.135.129
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.129
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:14:13.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:14:13.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.049362 30568 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:14:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.676090 30570 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.676771 30570 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:13.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.678131 30570 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.718487 30543 catalog_manager.cc:1269] Loaded cluster ID: b4bbe5694ba54b4bb19b829763a5a65f
09:14:13.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.718778 30543 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: loading cluster ID for follower catalog manager: success
09:14:13.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.725728 30543 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: acquiring CA information for follower catalog manager: success
09:14:13.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.739115 30543 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:14:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.740592 30570 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.741554 30570 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:14:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.741864 30570 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:14:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.742107 30570 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:14:13.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.742848 30570 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.135.129
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.810774 30570 tablet_server_runner.cc:78] Tablet server non-default flags:
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/wal
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.129:0
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data/info.pb
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.129
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:13.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.135.129
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/logs
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:20 UTC on 5fd53c4cbb9d
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8773
09:14:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.812611 30570 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:13.814919 30570 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.847553 30576 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:15.236831 30575 debug-util.cc:398] Leaking SignalData structure 0x7b08000349e0 after lost signal to thread 30570
09:14:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.024482 30575 kernel_stack_watchdog.cc:198] Thread 30570 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 388ms:
09:14:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:14:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:14:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:14:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:14:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:13.849407 30577 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.025568 30570 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.179s user 0.814s sys 1.221s
09:14:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.025899 30570 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.179s user 0.814s sys 1.221s
09:14:16.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.037173 30570 server_base.cc:1047] running on GCE node
09:14:16.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.038385 30582 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.039815 30570 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:16.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.046762 30570 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.048215 30570 hybrid_clock.cc:648] HybridClock initialized: now 1761729256048154 us; error 39 us; skew 500 ppm
09:14:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.049343 30570 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.074765 30570 webserver.cc:492] Webserver started at http://127.29.135.129:40989/ using document root <none> and password file <none>
09:14:16.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.076121 30570 fs_manager.cc:362] Metadata directory not provided
09:14:16.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.076422 30570 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.077020 30570 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:14:16.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.083395 30570 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data/instance:
09:14:16.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9d202cd45da486cbe20e2093811658e"
09:14:16.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:16 on dist-test-slave-td26"
09:14:16.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.085032 30570 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/wal/instance:
09:14:16.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9d202cd45da486cbe20e2093811658e"
09:14:16.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:16 on dist-test-slave-td26"
09:14:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.094914 30570 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
09:14:16.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.102819 30587 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:16.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.104552 30570 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
09:14:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.104985 30570 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data,/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/wal
09:14:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9d202cd45da486cbe20e2093811658e"
09:14:16.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:16 on dist-test-slave-td26"
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.105437 30570 fs_report.cc:389] FS layout report
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/wal
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/wal
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data/data
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:16.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.193274 30570 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:16.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.195498 30570 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:16.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.196245 30570 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:16.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.250208 30570 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:14:16.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.250741 30570 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.001s
09:14:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.251147 30570 ts_tablet_manager.cc:616] Registered 0 tablets
09:14:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.251474 30570 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:14:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.545626 30570 rpc_server.cc:307] RPC server started. Bound to: 127.29.135.129:36359
09:14:16.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.546151 30713 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.135.129:36359 every 8 connection(s)
09:14:16.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.548668 30570 server_base.cc:1179] Dumped server information to /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/data/info.pb
09:14:16.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.556885 30238 external_mini_cluster.cc:1428] Started /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu as pid 30570
09:14:16.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.557536 30238 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-0/wal/instance
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.570156 30238 external_mini_cluster.cc:1366] Running /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/wal
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/logs
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data/info.pb
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.130:0
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.135.130
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.130
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:14:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:14:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.572417 30714 heartbeater.cc:344] Connected to a master server at 127.29.135.188:44461
09:14:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.572909 30714 heartbeater.cc:461] Registering TS with master...
09:14:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.574347 30714 heartbeater.cc:507] Master 127.29.135.188:44461 requested a full tablet report, sending...
09:14:16.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.577682 30458 ts_manager.cc:194] Registered new tserver with Master: c9d202cd45da486cbe20e2093811658e (127.29.135.129:36359)
09:14:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.584174 30458 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.135.129:38987
09:14:16.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.590121 30717 heartbeater.cc:344] Connected to a master server at 127.29.135.190:41627
09:14:16.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.591603 30715 heartbeater.cc:344] Connected to a master server at 127.29.135.189:35713
09:14:16.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.592149 30717 heartbeater.cc:461] Registering TS with master...
09:14:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.592172 30715 heartbeater.cc:461] Registering TS with master...
09:14:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.592846 30717 heartbeater.cc:507] Master 127.29.135.190:41627 requested a full tablet report, sending...
09:14:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.592970 30715 heartbeater.cc:507] Master 127.29.135.189:35713 requested a full tablet report, sending...
09:14:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.596110 30368 ts_manager.cc:194] Registered new tserver with Master: c9d202cd45da486cbe20e2093811658e (127.29.135.129:36359)
09:14:16.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:16.620491 30283 ts_manager.cc:194] Registered new tserver with Master: c9d202cd45da486cbe20e2093811658e (127.29.135.129:36359)
09:14:16.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.926561 30719 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:16.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.927086 30719 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:16.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.927724 30719 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.979048 30719 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:16.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.979980 30719 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:14:16.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.980283 30719 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:14:16.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.980545 30719 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:14:16.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:16.981298 30719 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.135.130
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:17.018105 30719 tablet_server_runner.cc:78] Tablet server non-default flags:
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/wal
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.130:0
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data/info.pb
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.130
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:17.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.135.130
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/logs
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:20 UTC on 5fd53c4cbb9d
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8773
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:17.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:17.019342 30719 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:17.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:17.020890 30719 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:17.033902 30727 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:17.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:17.588425 30714 heartbeater.cc:499] Master 127.29.135.188:44461 was elected leader, sending a full tablet report...
09:14:18.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:17.041730 30719 server_base.cc:1047] running on GCE node
09:14:18.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:17.040567 30730 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:18.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:17.039208 30728 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.205682 30719 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.211944 30719 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:18.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.213392 30719 hybrid_clock.cc:648] HybridClock initialized: now 1761729258213353 us; error 61 us; skew 500 ppm
09:14:18.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.214241 30719 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:18.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.220165 30719 webserver.cc:492] Webserver started at http://127.29.135.130:45561/ using document root <none> and password file <none>
09:14:18.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.221143 30719 fs_manager.cc:362] Metadata directory not provided
09:14:18.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.221390 30719 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:18.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.221776 30719 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:14:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.226095 30719 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data/instance:
09:14:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7"
09:14:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:18 on dist-test-slave-td26"
09:14:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.227175 30719 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/wal/instance:
09:14:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7"
09:14:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:18 on dist-test-slave-td26"
09:14:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.235448 30719 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
09:14:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.241103 30737 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:18.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.242226 30719 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:14:18.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.242509 30719 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data,/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/wal
09:14:18.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7"
09:14:18.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:18 on dist-test-slave-td26"
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.242797 30719 fs_report.cc:389] FS layout report
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/wal
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/wal
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data/data
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.289856 30719 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.291383 30719 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:18.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.291776 30719 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.301535 30719 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:14:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.301741 30719 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:14:18.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.301951 30719 ts_tablet_manager.cc:616] Registered 0 tablets
09:14:18.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.302138 30719 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:14:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.478912 30719 rpc_server.cc:307] RPC server started. Bound to: 127.29.135.130:40585
09:14:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.479383 30862 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.135.130:40585 every 8 connection(s)
09:14:18.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.481426 30719 server_base.cc:1179] Dumped server information to /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/data/info.pb
09:14:18.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.489843 30238 external_mini_cluster.cc:1428] Started /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu as pid 30719
09:14:18.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.490422 30238 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-1/wal/instance
09:14:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.498222 30863 heartbeater.cc:344] Connected to a master server at 127.29.135.188:44461
09:14:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.498769 30863 heartbeater.cc:461] Registering TS with master...
09:14:18.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.500190 30863 heartbeater.cc:507] Master 127.29.135.188:44461 requested a full tablet report, sending...
09:14:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.503504 30457 ts_manager.cc:194] Registered new tserver with Master: 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585)
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.505834 30457 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.135.130:54853
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.506870 30865 heartbeater.cc:344] Connected to a master server at 127.29.135.190:41627
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.507031 30238 external_mini_cluster.cc:1366] Running /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/wal
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:18.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/logs
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data/info.pb
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.131:0
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.135.131
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.131
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.507216 30865 heartbeater.cc:461] Registering TS with master...
09:14:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.508083 30865 heartbeater.cc:507] Master 127.29.135.190:41627 requested a full tablet report, sending...
09:14:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.509441 30864 heartbeater.cc:344] Connected to a master server at 127.29.135.189:35713
09:14:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.509779 30864 heartbeater.cc:461] Registering TS with master...
09:14:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.510381 30864 heartbeater.cc:507] Master 127.29.135.189:35713 requested a full tablet report, sending...
09:14:18.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.511180 30283 ts_manager.cc:194] Registered new tserver with Master: 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585)
09:14:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.512681 30368 ts_manager.cc:194] Registered new tserver with Master: 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585)
09:14:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.809368 30868 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:14:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.809834 30868 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:14:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.810366 30868 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:14:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.843570 30868 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:14:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.844252 30868 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
09:14:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.844465 30868 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
09:14:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.844621 30868 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
09:14:18.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.845566 30868 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.135.131
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.881888 30868 tablet_server_runner.cc:78] Tablet server non-default flags:
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.135.148:43633
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/wal
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.135.131:0
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data/info.pb
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.135.131
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.135.190:41627,127.29.135.189:35713,127.29.135.188:44461
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.135.131
09:14:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/logs
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 08:43:20 UTC on 5fd53c4cbb9d
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8773
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:14:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.883124 30868 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:14:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:18.884624 30868 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:14:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.896198 30874 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:19.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:19.510819 30863 heartbeater.cc:499] Master 127.29.135.188:44461 was elected leader, sending a full tablet report...
09:14:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.897634 30875 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:18.901139 30877 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:14:20.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:20.179723 30876 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1277 milliseconds
09:14:20.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.179827 30868 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:14:20.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.180974 30868 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:14:20.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.183876 30868 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:14:20.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.185289 30868 hybrid_clock.cc:648] HybridClock initialized: now 1761729260185236 us; error 70 us; skew 500 ppm
09:14:20.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.186163 30868 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:14:20.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.192314 30868 webserver.cc:492] Webserver started at http://127.29.135.131:37301/ using document root <none> and password file <none>
09:14:20.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.193243 30868 fs_manager.cc:362] Metadata directory not provided
09:14:20.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.193454 30868 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:14:20.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.193840 30868 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
09:14:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.198058 30868 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data/instance:
09:14:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db1ab16d1eca45fd992989999e24a5be"
09:14:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:20 on dist-test-slave-td26"
09:14:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.199083 30868 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/wal/instance:
09:14:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db1ab16d1eca45fd992989999e24a5be"
09:14:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:20 on dist-test-slave-td26"
09:14:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.208586 30868 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
09:14:20.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.215227 30885 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:14:20.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.216303 30868 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.216656 30868 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data,/tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/wal
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db1ab16d1eca45fd992989999e24a5be"
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 09:14:20 on dist-test-slave-td26"
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.217022 30868 fs_report.cc:389] FS layout report
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/wal
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/wal
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data/data
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:14:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:14:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:14:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:14:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:14:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:14:20.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:14:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.277958 30868 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:14:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.279462 30868 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:14:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.279958 30868 kserver.cc:163] Server-wide thread pool size limit: 3276
09:14:20.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.290272 30868 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:14:20.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.290503 30868 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:14:20.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.290705 30868 ts_tablet_manager.cc:616] Registered 0 tablets
09:14:20.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.290839 30868 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:14:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.456053 30868 rpc_server.cc:307] RPC server started. Bound to: 127.29.135.131:37129
09:14:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.456496 31009 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.135.131:37129 every 8 connection(s)
09:14:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.458654 30868 server_base.cc:1179] Dumped server information to /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/data/info.pb
09:14:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.465521 30238 external_mini_cluster.cc:1428] Started /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu as pid 30868
09:14:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.466032 30238 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task8A6J2s/test-tmp/mini-kudu-cluster5957031018788633169/ts-2/wal/instance
09:14:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.468924 31012 heartbeater.cc:344] Connected to a master server at 127.29.135.190:41627
09:14:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.469383 31012 heartbeater.cc:461] Registering TS with master...
09:14:20.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.471132 31012 heartbeater.cc:507] Master 127.29.135.190:41627 requested a full tablet report, sending...
09:14:20.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.474252 30283 ts_manager.cc:194] Registered new tserver with Master: db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129)
09:14:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.480249 31010 heartbeater.cc:344] Connected to a master server at 127.29.135.188:44461
09:14:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.480640 31010 heartbeater.cc:461] Registering TS with master...
09:14:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.481578 31010 heartbeater.cc:507] Master 127.29.135.188:44461 requested a full tablet report, sending...
09:14:20.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.484006 30457 ts_manager.cc:194] Registered new tserver with Master: db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129)
09:14:20.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.485728 30457 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.135.131:44379
09:14:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.489825 31011 heartbeater.cc:344] Connected to a master server at 127.29.135.189:35713
09:14:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.490127 31011 heartbeater.cc:461] Registering TS with master...
09:14:20.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.490705 31011 heartbeater.cc:507] Master 127.29.135.189:35713 requested a full tablet report, sending...
09:14:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.493682 30367 ts_manager.cc:194] Registered new tserver with Master: db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129)
09:14:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.515651 30238 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
09:14:20.529 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:14:20.529 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:14:20.530 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.135.190"
port: 41627
}
bound_http_address {
host: "127.29.135.190"
port: 33205
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.135.189"
port: 35713
}
bound_http_address {
host: "127.29.135.189"
port: 44509
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.135.188"
port: 44461
}
bound_http_address {
host: "127.29.135.188"
port: 38499
}
}
}
09:14:20.530 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:14:20.531 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.29.135.129"
port: 36359
}
bound_http_address {
host: "127.29.135.129"
port: 40989
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.29.135.130"
port: 40585
}
bound_http_address {
host: "127.29.135.130"
port: 45561
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.29.135.131"
port: 37129
}
bound_http_address {
host: "127.29.135.131"
port: 37301
}
}
}
09:14:20.531 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:14:20.533 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:14:20.534 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:20.535 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.135.189:35713(127.29.135.189:35713)] connecting to peer
09:14:20.536 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.135.190:41627(127.29.135.190:41627)] connecting to peer
09:14:20.537 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.29.135.189:35713(127.29.135.189:35713)] Successfully connected to peer
09:14:20.539 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.135.188:44461(127.29.135.188:44461)] Successfully connected to peer
09:14:20.541 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.135.190:41627(127.29.135.190:41627)] Successfully connected to peer
09:14:20.541 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.135.189
09:14:20.542 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.135.188
09:14:20.545 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.29.135.190
09:14:20.557 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7a826813, L:/127.0.0.1:52516 - R:/127.29.135.189:35713] using SASL/PLAIN
09:14:20.567 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb58f22c4, L:/127.0.0.1:43610 - R:/127.29.135.188:44461] using SASL/PLAIN
09:14:20.568 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x470803e0, L:/127.0.0.1:40238 - R:/127.29.135.190:41627] using SASL/PLAIN
09:14:20.570 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:14:20.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=37), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [29ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [35ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: OK, deferred=null) after lookup
09:14:20.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.572754 30457 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43610:
09:14:20.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback"
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:20.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:20.576371 30457 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:14:20.609 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 467c81dd39584ccb95d4ec6a6d4c0d78
09:14:20.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.660593 30797 tablet_service.cc:1505] Processing CreateTablet for tablet 09fb3f24486042db8e27a473de715973 (DEFAULT_TABLE table=txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:14:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.662685 30797 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 09fb3f24486042db8e27a473de715973. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:20.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.663111 30649 tablet_service.cc:1505] Processing CreateTablet for tablet 30527506731f4ccb9dd46fbef2e4136e (DEFAULT_TABLE table=txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:14:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.664556 30649 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 30527506731f4ccb9dd46fbef2e4136e. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:20.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.665161 30944 tablet_service.cc:1505] Processing CreateTablet for tablet 09fb3f24486042db8e27a473de715973 (DEFAULT_TABLE table=txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:14:20.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.667003 30944 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 09fb3f24486042db8e27a473de715973. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.670019 30648 tablet_service.cc:1505] Processing CreateTablet for tablet 09fb3f24486042db8e27a473de715973 (DEFAULT_TABLE table=txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
09:14:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.670475 30945 tablet_service.cc:1505] Processing CreateTablet for tablet 30527506731f4ccb9dd46fbef2e4136e (DEFAULT_TABLE table=txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:14:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.671026 30648 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 09fb3f24486042db8e27a473de715973. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:20.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.671658 30945 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 30527506731f4ccb9dd46fbef2e4136e. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.673481 30798 tablet_service.cc:1505] Processing CreateTablet for tablet 30527506731f4ccb9dd46fbef2e4136e (DEFAULT_TABLE table=txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
09:14:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.678107 30798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 30527506731f4ccb9dd46fbef2e4136e. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:20.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.699116 31037 tablet_bootstrap.cc:492] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Bootstrap starting.
09:14:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.710134 31037 tablet_bootstrap.cc:654] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Neither blocks nor log segments found. Creating new log.
09:14:20.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.710757 31039 tablet_bootstrap.cc:492] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be: Bootstrap starting.
09:14:20.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.712725 31037 log.cc:826] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Log is configured to *not* fsync() on all Append() calls
09:14:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.720059 31039 tablet_bootstrap.cc:654] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be: Neither blocks nor log segments found. Creating new log.
09:14:20.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.722715 31039 log.cc:826] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be: Log is configured to *not* fsync() on all Append() calls
09:14:20.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.722880 31038 tablet_bootstrap.cc:492] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e: Bootstrap starting.
09:14:20.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.725183 31037 tablet_bootstrap.cc:492] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7: No bootstrap required, opened a new log
09:14:20.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.725929 31037 ts_tablet_manager.cc:1403] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Time spent bootstrapping tablet: real 0.027s user 0.013s sys 0.008s
09:14:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.734022 31039 tablet_bootstrap.cc:492] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be: No bootstrap required, opened a new log
09:14:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.734575 31039 ts_tablet_manager.cc:1403] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be: Time spent bootstrapping tablet: real 0.026s user 0.006s sys 0.015s
09:14:20.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.735255 31038 tablet_bootstrap.cc:654] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e: Neither blocks nor log segments found. Creating new log.
09:14:20.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.737671 31038 log.cc:826] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e: Log is configured to *not* fsync() on all Append() calls
09:14:20.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.742748 31038 tablet_bootstrap.cc:492] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e: No bootstrap required, opened a new log
09:14:20.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.743221 31038 ts_tablet_manager.cc:1403] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e: Time spent bootstrapping tablet: real 0.021s user 0.009s sys 0.003s
09:14:20.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.755607 31037 raft_consensus.cc:359] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.756594 31037 raft_consensus.cc:385] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:20.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.756944 31037 raft_consensus.cc:740] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ea4ee372f2f44c9878debc9a6ba4bf7, State: Initialized, Role: FOLLOWER
09:14:20.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.757894 31037 consensus_queue.cc:260] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.761888 31037 ts_tablet_manager.cc:1434] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Time spent starting tablet: real 0.036s user 0.029s sys 0.007s
09:14:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.763168 31039 raft_consensus.cc:359] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.764093 31039 raft_consensus.cc:385] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.764397 31039 raft_consensus.cc:740] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db1ab16d1eca45fd992989999e24a5be, State: Initialized, Role: FOLLOWER
09:14:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.765332 31039 consensus_queue.cc:260] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.767117 31038 raft_consensus.cc:359] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.768218 31038 raft_consensus.cc:385] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:20.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.768487 31038 raft_consensus.cc:740] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9d202cd45da486cbe20e2093811658e, State: Initialized, Role: FOLLOWER
09:14:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.769354 31038 consensus_queue.cc:260] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.770712 31037 tablet_bootstrap.cc:492] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7: Bootstrap starting.
09:14:20.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.780818 31010 heartbeater.cc:499] Master 127.29.135.188:44461 was elected leader, sending a full tablet report...
09:14:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.785015 31039 ts_tablet_manager.cc:1434] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be: Time spent starting tablet: real 0.050s user 0.026s sys 0.020s
09:14:20.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.787539 31037 tablet_bootstrap.cc:654] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7: Neither blocks nor log segments found. Creating new log.
09:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.790246 31039 tablet_bootstrap.cc:492] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be: Bootstrap starting.
09:14:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.790938 31038 ts_tablet_manager.cc:1434] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e: Time spent starting tablet: real 0.047s user 0.030s sys 0.008s
09:14:20.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.798151 31039 tablet_bootstrap.cc:654] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be: Neither blocks nor log segments found. Creating new log.
09:14:20.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.801771 31038 tablet_bootstrap.cc:492] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e: Bootstrap starting.
09:14:20.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.811676 31038 tablet_bootstrap.cc:654] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e: Neither blocks nor log segments found. Creating new log.
09:14:20.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.817589 31037 tablet_bootstrap.cc:492] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7: No bootstrap required, opened a new log
09:14:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.817788 31039 tablet_bootstrap.cc:492] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be: No bootstrap required, opened a new log
09:14:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.818064 31037 ts_tablet_manager.cc:1403] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7: Time spent bootstrapping tablet: real 0.048s user 0.008s sys 0.022s
09:14:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.818197 31039 ts_tablet_manager.cc:1403] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be: Time spent bootstrapping tablet: real 0.028s user 0.012s sys 0.012s
09:14:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.818261 31038 tablet_bootstrap.cc:492] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e: No bootstrap required, opened a new log
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.818668 31038 ts_tablet_manager.cc:1403] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e: Time spent bootstrapping tablet: real 0.017s user 0.008s sys 0.005s
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.819388 31043 raft_consensus.cc:493] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.819977 31043 raft_consensus.cc:515] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.820750 31037 raft_consensus.cc:359] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.820874 31039 raft_consensus.cc:359] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.821467 31037 raft_consensus.cc:385] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.821507 31039 raft_consensus.cc:385] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.821403 31038 raft_consensus.cc:359] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.821760 31039 raft_consensus.cc:740] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db1ab16d1eca45fd992989999e24a5be, State: Initialized, Role: FOLLOWER
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.821745 31037 raft_consensus.cc:740] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ea4ee372f2f44c9878debc9a6ba4bf7, State: Initialized, Role: FOLLOWER
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.822038 31038 raft_consensus.cc:385] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.822314 31038 raft_consensus.cc:740] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9d202cd45da486cbe20e2093811658e, State: Initialized, Role: FOLLOWER
09:14:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.822479 31039 consensus_queue.cc:260] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.822443 31037 consensus_queue.cc:260] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.822942 31038 consensus_queue.cc:260] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.824990 31039 ts_tablet_manager.cc:1434] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be: Time spent starting tablet: real 0.007s user 0.008s sys 0.000s
09:14:20.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.825109 31038 ts_tablet_manager.cc:1434] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e: Time spent starting tablet: real 0.006s user 0.008s sys 0.000s
09:14:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.837394 31043 leader_election.cc:290] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9d202cd45da486cbe20e2093811658e (127.29.135.129:36359), db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129)
09:14:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.839998 31037 ts_tablet_manager.cc:1434] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7: Time spent starting tablet: real 0.022s user 0.007s sys 0.010s
09:14:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:20.846771 30720 tablet.cc:2378] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:14:20.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.853833 30669 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "09fb3f24486042db8e27a473de715973" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9d202cd45da486cbe20e2093811658e" is_pre_election: true
09:14:20.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.854789 30669 raft_consensus.cc:2468] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 0.
09:14:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.856381 30738 leader_election.cc:304] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ea4ee372f2f44c9878debc9a6ba4bf7, c9d202cd45da486cbe20e2093811658e; no voters:
09:14:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.857990 31049 raft_consensus.cc:2804] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:14:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.858430 31049 raft_consensus.cc:493] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:14:20.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.858758 31049 raft_consensus.cc:3060] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Advancing to term 1
09:14:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.865031 31049 raft_consensus.cc:515] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.865882 30965 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "09fb3f24486042db8e27a473de715973" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db1ab16d1eca45fd992989999e24a5be" is_pre_election: true
09:14:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.866611 31049 leader_election.cc:290] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 election: Requested vote from peers c9d202cd45da486cbe20e2093811658e (127.29.135.129:36359), db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129)
09:14:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.866591 30965 raft_consensus.cc:2468] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 0.
09:14:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.867504 30669 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "09fb3f24486042db8e27a473de715973" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9d202cd45da486cbe20e2093811658e"
09:14:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.868062 30669 raft_consensus.cc:3060] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Advancing to term 1
09:14:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.867966 30964 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "09fb3f24486042db8e27a473de715973" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db1ab16d1eca45fd992989999e24a5be"
09:14:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.868505 30964 raft_consensus.cc:3060] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Advancing to term 1
09:14:20.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.875212 30669 raft_consensus.cc:2468] T 09fb3f24486042db8e27a473de715973 P c9d202cd45da486cbe20e2093811658e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 1.
09:14:20.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.875209 30964 raft_consensus.cc:2468] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 1.
09:14:20.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.876168 30738 leader_election.cc:304] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ea4ee372f2f44c9878debc9a6ba4bf7, c9d202cd45da486cbe20e2093811658e; no voters:
09:14:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.876803 31049 raft_consensus.cc:2804] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 1 FOLLOWER]: Leader election won for term 1
09:14:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.877234 31049 raft_consensus.cc:697] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 1 LEADER]: Becoming Leader. State: Replica: 2ea4ee372f2f44c9878debc9a6ba4bf7, State: Running, Role: LEADER
09:14:20.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.878197 31049 consensus_queue.cc:237] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.883056 31045 raft_consensus.cc:493] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:20.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.883482 31045 raft_consensus.cc:515] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.886085 31045 leader_election.cc:290] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129), 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585)
09:14:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.896230 30457 catalog_manager.cc:5649] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130). New cstate: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } health_report { overall_health: UNKNOWN } } }
09:14:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.898787 30964 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "30527506731f4ccb9dd46fbef2e4136e" candidate_uuid: "c9d202cd45da486cbe20e2093811658e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db1ab16d1eca45fd992989999e24a5be" is_pre_election: true
09:14:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.899340 30964 raft_consensus.cc:2468] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9d202cd45da486cbe20e2093811658e in term 0.
09:14:20.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.900333 30592 leader_election.cc:304] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9d202cd45da486cbe20e2093811658e, db1ab16d1eca45fd992989999e24a5be; no voters:
09:14:20.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.901010 31045 raft_consensus.cc:2804] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Leader pre-election won for term 1
09:14:20.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.901346 31045 raft_consensus.cc:493] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:14:20.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.901652 31045 raft_consensus.cc:3060] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Advancing to term 1
09:14:20.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.901962 30818 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "30527506731f4ccb9dd46fbef2e4136e" candidate_uuid: "c9d202cd45da486cbe20e2093811658e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" is_pre_election: true
09:14:20.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.902724 30818 raft_consensus.cc:2468] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9d202cd45da486cbe20e2093811658e in term 0.
09:14:20.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.907832 31045 raft_consensus.cc:515] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.909402 31045 leader_election.cc:290] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [CANDIDATE]: Term 1 election: Requested vote from peers db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129), 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585)
09:14:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.910375 30964 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "30527506731f4ccb9dd46fbef2e4136e" candidate_uuid: "c9d202cd45da486cbe20e2093811658e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db1ab16d1eca45fd992989999e24a5be"
09:14:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.910888 30964 raft_consensus.cc:3060] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Advancing to term 1
09:14:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.911062 30818 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "30527506731f4ccb9dd46fbef2e4136e" candidate_uuid: "c9d202cd45da486cbe20e2093811658e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7"
09:14:20.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.911594 30818 raft_consensus.cc:3060] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Advancing to term 1
09:14:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.917389 30964 raft_consensus.cc:2468] T 30527506731f4ccb9dd46fbef2e4136e P db1ab16d1eca45fd992989999e24a5be [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9d202cd45da486cbe20e2093811658e in term 1.
09:14:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.918340 30592 leader_election.cc:304] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9d202cd45da486cbe20e2093811658e, db1ab16d1eca45fd992989999e24a5be; no voters:
09:14:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.918989 31045 raft_consensus.cc:2804] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 1 FOLLOWER]: Leader election won for term 1
09:14:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.919597 30818 raft_consensus.cc:2468] T 30527506731f4ccb9dd46fbef2e4136e P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9d202cd45da486cbe20e2093811658e in term 1.
09:14:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.920703 31045 raft_consensus.cc:697] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [term 1 LEADER]: Becoming Leader. State: Replica: c9d202cd45da486cbe20e2093811658e, State: Running, Role: LEADER
09:14:20.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.921687 31045 consensus_queue.cc:237] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } }
09:14:20.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:20.934549 30457 catalog_manager.cc:5649] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e reported cstate change: term changed from 0 to 1, leader changed from <none> to c9d202cd45da486cbe20e2093811658e (127.29.135.129). New cstate: current_term: 1 leader_uuid: "c9d202cd45da486cbe20e2093811658e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } health_report { overall_health: UNKNOWN } } }
09:14:20.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:20.993112 31014 tablet.cc:2378] T 09fb3f24486042db8e27a473de715973 P db1ab16d1eca45fd992989999e24a5be: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:14:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:21.009827 30867 tablet.cc:2378] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.031594 30457 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43622:
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:14:21.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:21.033716 30457 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:14:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.069604 30648 tablet_service.cc:1505] Processing CreateTablet for tablet 340913df5e7a4d35bab2c47335b2fc17 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=713eb997bab6492f9beb8cb357fd218d]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:14:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.071225 30648 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 340913df5e7a4d35bab2c47335b2fc17. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.075573 30798 tablet_service.cc:1505] Processing CreateTablet for tablet 340913df5e7a4d35bab2c47335b2fc17 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=713eb997bab6492f9beb8cb357fd218d]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:14:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.077178 30798 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 340913df5e7a4d35bab2c47335b2fc17. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.077973 30944 tablet_service.cc:1505] Processing CreateTablet for tablet 340913df5e7a4d35bab2c47335b2fc17 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=713eb997bab6492f9beb8cb357fd218d]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
09:14:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.079401 30944 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 340913df5e7a4d35bab2c47335b2fc17. 1 dirs total, 0 dirs full, 0 dirs failed
09:14:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.091847 31037 tablet_bootstrap.cc:492] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Bootstrap starting.
09:14:21.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.092566 31039 tablet_bootstrap.cc:492] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: Bootstrap starting.
09:14:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.097985 31037 tablet_bootstrap.cc:654] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Neither blocks nor log segments found. Creating new log.
09:14:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.098253 31039 tablet_bootstrap.cc:654] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: Neither blocks nor log segments found. Creating new log.
09:14:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.102793 31038 tablet_bootstrap.cc:492] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: Bootstrap starting.
09:14:21.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.106473 31039 tablet_bootstrap.cc:492] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: No bootstrap required, opened a new log
09:14:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.106915 31039 ts_tablet_manager.cc:1403] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: Time spent bootstrapping tablet: real 0.015s user 0.010s sys 0.000s
09:14:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.109561 31039 raft_consensus.cc:359] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.110133 31038 tablet_bootstrap.cc:654] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: Neither blocks nor log segments found. Creating new log.
09:14:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.110252 31039 raft_consensus.cc:385] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.110533 31039 raft_consensus.cc:740] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db1ab16d1eca45fd992989999e24a5be, State: Initialized, Role: FOLLOWER
09:14:21.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.111224 31039 consensus_queue.cc:260] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.112658 31044 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.120986 31044 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: This TxnStatusTablet replica's current role is: FOLLOWER
09:14:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.123065 31039 ts_tablet_manager.cc:1434] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: Time spent starting tablet: real 0.016s user 0.007s sys 0.000s
09:14:21.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.125185 31037 tablet_bootstrap.cc:492] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: No bootstrap required, opened a new log
09:14:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.127964 31037 ts_tablet_manager.cc:1403] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Time spent bootstrapping tablet: real 0.036s user 0.014s sys 0.015s
09:14:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.128521 31038 tablet_bootstrap.cc:492] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: No bootstrap required, opened a new log
09:14:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.128944 31038 ts_tablet_manager.cc:1403] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: Time spent bootstrapping tablet: real 0.026s user 0.012s sys 0.005s
09:14:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.130497 31037 raft_consensus.cc:359] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.131081 31037 raft_consensus.cc:385] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:21.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.131335 31037 raft_consensus.cc:740] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2ea4ee372f2f44c9878debc9a6ba4bf7, State: Initialized, Role: FOLLOWER
09:14:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.131920 31037 consensus_queue.cc:260] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.132105 31038 raft_consensus.cc:359] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.132766 31038 raft_consensus.cc:385] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:14:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.133077 31038 raft_consensus.cc:740] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9d202cd45da486cbe20e2093811658e, State: Initialized, Role: FOLLOWER
09:14:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.133513 31049 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.134207 31049 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: This TxnStatusTablet replica's current role is: FOLLOWER
09:14:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.133739 31038 consensus_queue.cc:260] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.134761 31037 ts_tablet_manager.cc:1434] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: Time spent starting tablet: real 0.006s user 0.005s sys 0.001s
09:14:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.134949 31045 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.135641 31045 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: This TxnStatusTablet replica's current role is: FOLLOWER
09:14:21.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.137696 31038 ts_tablet_manager.cc:1434] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: Time spent starting tablet: real 0.008s user 0.006s sys 0.000s
09:14:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.242344 31049 raft_consensus.cc:493] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.242720 31049 raft_consensus.cc:515] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.244037 31049 leader_election.cc:290] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9d202cd45da486cbe20e2093811658e (127.29.135.129:36359), db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129)
09:14:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.244832 30669 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "340913df5e7a4d35bab2c47335b2fc17" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9d202cd45da486cbe20e2093811658e" is_pre_election: true
09:14:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.245245 30964 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "340913df5e7a4d35bab2c47335b2fc17" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db1ab16d1eca45fd992989999e24a5be" is_pre_election: true
09:14:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.245498 30669 raft_consensus.cc:2468] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 0.
09:14:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.245849 30964 raft_consensus.cc:2468] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 0.
09:14:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.246420 30738 leader_election.cc:304] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ea4ee372f2f44c9878debc9a6ba4bf7, c9d202cd45da486cbe20e2093811658e; no voters:
09:14:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.247126 31049 raft_consensus.cc:2804] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:14:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.247435 31049 raft_consensus.cc:493] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:14:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.247736 31049 raft_consensus.cc:3060] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 0 FOLLOWER]: Advancing to term 1
09:14:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.251868 31049 raft_consensus.cc:515] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.253330 31049 leader_election.cc:290] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 election: Requested vote from peers c9d202cd45da486cbe20e2093811658e (127.29.135.129:36359), db1ab16d1eca45fd992989999e24a5be (127.29.135.131:37129)
09:14:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.254182 30669 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "340913df5e7a4d35bab2c47335b2fc17" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9d202cd45da486cbe20e2093811658e"
09:14:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.254436 30964 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "340913df5e7a4d35bab2c47335b2fc17" candidate_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db1ab16d1eca45fd992989999e24a5be"
09:14:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.254778 30669 raft_consensus.cc:3060] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [term 0 FOLLOWER]: Advancing to term 1
09:14:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.254951 30964 raft_consensus.cc:3060] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [term 0 FOLLOWER]: Advancing to term 1
09:14:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.262241 30669 raft_consensus.cc:2468] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 1.
09:14:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.263476 30964 raft_consensus.cc:2468] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2ea4ee372f2f44c9878debc9a6ba4bf7 in term 1.
09:14:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.263525 30738 leader_election.cc:304] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2ea4ee372f2f44c9878debc9a6ba4bf7, c9d202cd45da486cbe20e2093811658e; no voters:
09:14:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.264348 31049 raft_consensus.cc:2804] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 1 FOLLOWER]: Leader election won for term 1
09:14:21.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.264775 31049 raft_consensus.cc:697] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [term 1 LEADER]: Becoming Leader. State: Replica: 2ea4ee372f2f44c9878debc9a6ba4bf7, State: Running, Role: LEADER
09:14:21.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.265533 31049 consensus_queue.cc:237] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } }
09:14:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.272033 31043 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: TxnStatusTablet state changed. Reason: New leader 2ea4ee372f2f44c9878debc9a6ba4bf7. Latest consensus state: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.272666 31043 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: This TxnStatusTablet replica's current role is: LEADER
09:14:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.276011 31070 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
09:14:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.275537 30454 catalog_manager.cc:5649] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 reported cstate change: term changed from 0 to 1, leader changed from <none> to 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130). New cstate: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } health_report { overall_health: UNKNOWN } } }
09:14:21.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.276610 31070 txn_status_manager.cc:930] Loading transaction status metadata into memory...
09:14:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.333920 30964 raft_consensus.cc:1275] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be [term 1 FOLLOWER]: Refusing update from remote peer 2ea4ee372f2f44c9878debc9a6ba4bf7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:14:21.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.334367 30669 raft_consensus.cc:1275] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e [term 1 FOLLOWER]: Refusing update from remote peer 2ea4ee372f2f44c9878debc9a6ba4bf7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:14:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.335453 31049 consensus_queue.cc:1048] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.336207 31043 consensus_queue.cc:1048] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:21.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.373787 31073 mvcc.cc:204] Tried to move back new op lower bound from 7216043054408142848 to 7216043054149894144. Current Snapshot: MvccSnapshot[applied={T|T < 7216043054408142848}]
09:14:21.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.374969 31045 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: TxnStatusTablet state changed. Reason: New leader 2ea4ee372f2f44c9878debc9a6ba4bf7. Latest consensus state: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.375893 31045 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: This TxnStatusTablet replica's current role is: FOLLOWER
09:14:21.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.382032 31049 consensus_queue.cc:1048] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:21.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.383662 31049 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.384341 31049 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: This TxnStatusTablet replica's current role is: LEADER
09:14:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.400560 31045 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.401549 31045 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P c9d202cd45da486cbe20e2093811658e: This TxnStatusTablet replica's current role is: FOLLOWER
09:14:21.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.402073 31081 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.402832 31081 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P 2ea4ee372f2f44c9878debc9a6ba4bf7: This TxnStatusTablet replica's current role is: LEADER
09:14:21.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.403256 31081 consensus_queue.cc:1048] T 09fb3f24486042db8e27a473de715973 P 2ea4ee372f2f44c9878debc9a6ba4bf7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.425064 31044 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: TxnStatusTablet state changed. Reason: New leader 2ea4ee372f2f44c9878debc9a6ba4bf7. Latest consensus state: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.425807 31044 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: This TxnStatusTablet replica's current role is: FOLLOWER
09:14:21.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.447428 31045 consensus_queue.cc:1048] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [LEADER]: Connected to new peer: Peer: permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.459234 31082 tablet_replica.cc:442] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9d202cd45da486cbe20e2093811658e" member_type: VOTER last_known_addr { host: "127.29.135.129" port: 36359 } } peers { permanent_uuid: "2ea4ee372f2f44c9878debc9a6ba4bf7" member_type: VOTER last_known_addr { host: "127.29.135.130" port: 40585 } } peers { permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 } } }
09:14:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.459952 31082 tablet_replica.cc:445] T 340913df5e7a4d35bab2c47335b2fc17 P db1ab16d1eca45fd992989999e24a5be: This TxnStatusTablet replica's current role is: FOLLOWER
09:14:21.472 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
09:14:21.475 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 467c81dd39584ccb95d4ec6a6d4c0d78
09:14:21.480 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 30527506731f4ccb9dd46fbef2e4136e for table 'txn_manager_fallback' with partition [0x00000000, 0x00000001)
09:14:21.480 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 09fb3f24486042db8e27a473de715973 for table 'txn_manager_fallback' with partition [0x00000001, 0x00000002)
09:14:21.481 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=30527506731f4ccb9dd46fbef2e4136e@[2ea4ee372f2f44c9878debc9a6ba4bf7(127.29.135.130:40585),c9d202cd45da486cbe20e2093811658e(127.29.135.129:36359)[L],db1ab16d1eca45fd992989999e24a5be(127.29.135.131:37129)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=09fb3f24486042db8e27a473de715973@[2ea4ee372f2f44c9878debc9a6ba4bf7(127.29.135.130:40585)[L],c9d202cd45da486cbe20e2093811658e(127.29.135.129:36359),db1ab16d1eca45fd992989999e24a5be(127.29.135.131:37129)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299998}]
09:14:21.481 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=5), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.29.135.188:44461, [3ms] Sub RPC GetTableLocations: received response from server master-127.29.135.188:44461: OK, deferred=null) after lookup
09:14:21.482 [DEBUG - kudu-nio-0] (Connection.java:748) [peer c9d202cd45da486cbe20e2093811658e(127.29.135.129:36359)] connecting to peer
09:14:21.485 [DEBUG - kudu-nio-1] (Connection.java:757) [peer c9d202cd45da486cbe20e2093811658e(127.29.135.129:36359)] Successfully connected to peer
09:14:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.488363 31045 consensus_queue.cc:1048] T 30527506731f4ccb9dd46fbef2e4136e P c9d202cd45da486cbe20e2093811658e [LEADER]: Connected to new peer: Peer: permanent_uuid: "db1ab16d1eca45fd992989999e24a5be" member_type: VOTER last_known_addr { host: "127.29.135.131" port: 37129 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:14:21.649 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6e3c73e1, L:/127.0.0.1:60832 - R:/127.29.135.129:36359] using TOKEN/null
09:14:21.750 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.135.188:44461
09:14:21.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:14:21.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:21.751703 30238 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu with pid 30415
09:14:21.791 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.135.188:44461(127.29.135.188:44461)] handling channelInactive
09:14:21.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state READY due to: connection closed
09:14:21.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:14:21.975 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:21.976 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:21.977 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:21.977 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.29.135.188:44461(127.29.135.188:44461) for tablet Kudu Master: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:21.994 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:21.995 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:21.995 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:21.996 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:21.998 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.014 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.015 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.015 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.015 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.017 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.018489 31096 raft_consensus.cc:493] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 7f85076611264064a8c1d15996e30278)
09:14:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.018741 31097 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 7f85076611264064a8c1d15996e30278)
09:14:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.019099 31096 raft_consensus.cc:515] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.019256 31097 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:22.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.022804 30377 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "317bacb8d32f45ee92c35da517f50844" is_pre_election: true
09:14:22.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.023106 30292 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "317bacb8d32f45ee92c35da517f50844" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" is_pre_election: true
09:14:22.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.023631 30377 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4f1ff44df6544f6ab375ae8c1fa999f6 in term 1.
09:14:22.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.023793 30292 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 317bacb8d32f45ee92c35da517f50844 in term 1.
09:14:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.025146 30355 leader_election.cc:304] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 317bacb8d32f45ee92c35da517f50844, 4f1ff44df6544f6ab375ae8c1fa999f6; no voters:
09:14:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.026016 30269 leader_election.cc:304] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 317bacb8d32f45ee92c35da517f50844, 4f1ff44df6544f6ab375ae8c1fa999f6; no voters:
09:14:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.027516 31097 leader_election.cc:290] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 317bacb8d32f45ee92c35da517f50844 (127.29.135.189:35713), 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461)
09:14:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.028076 31097 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:14:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.028445 31097 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 7f85076611264064a8c1d15996e30278)
09:14:22.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.028828 31097 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 1 FOLLOWER]: Advancing to term 2
09:14:22.035 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.036 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.036 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.036 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.041574 31096 leader_election.cc:290] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 4f1ff44df6544f6ab375ae8c1fa999f6 (127.29.135.190:41627), 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461)
09:14:22.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.042969 31096 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:14:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.044214 31097 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.045053 31096 raft_consensus.cc:493] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 7f85076611264064a8c1d15996e30278)
09:14:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.045552 31096 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 1 FOLLOWER]: Advancing to term 2
09:14:22.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.046101 30352 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111) [suppressed 6 similar messages]
09:14:22.049 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.052153 30267 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111) [suppressed 13 similar messages]
09:14:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.054248 30352 leader_election.cc:336] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.055 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.056 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.056 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.056 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.060653 31096 raft_consensus.cc:515] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:22.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.061663 30267 leader_election.cc:336] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.062072 30377 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "317bacb8d32f45ee92c35da517f50844"
09:14:22.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.062742 30377 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 4f1ff44df6544f6ab375ae8c1fa999f6 in current term 2: Already voted for candidate 317bacb8d32f45ee92c35da517f50844 in this term.
09:14:22.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.063211 31097 leader_election.cc:290] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 2 election: Requested vote from peers 317bacb8d32f45ee92c35da517f50844 (127.29.135.189:35713), 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461)
09:14:22.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.064182 31096 leader_election.cc:290] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [CANDIDATE]: Term 2 election: Requested vote from peers 4f1ff44df6544f6ab375ae8c1fa999f6 (127.29.135.190:41627), 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461)
09:14:22.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.064529 30292 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "317bacb8d32f45ee92c35da517f50844" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6"
09:14:22.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.065227 30292 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Leader election vote request: Denying vote to candidate 317bacb8d32f45ee92c35da517f50844 in current term 2: Already voted for candidate 4f1ff44df6544f6ab375ae8c1fa999f6 in this term.
09:14:22.069 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.068090 30352 leader_election.cc:336] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.068487 30352 leader_election.cc:304] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 317bacb8d32f45ee92c35da517f50844; no voters: 4f1ff44df6544f6ab375ae8c1fa999f6, 7f85076611264064a8c1d15996e30278
09:14:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.069550 31096 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
09:14:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.070084 30267 leader_election.cc:336] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.070411 30267 leader_election.cc:304] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f1ff44df6544f6ab375ae8c1fa999f6; no voters: 317bacb8d32f45ee92c35da517f50844, 7f85076611264064a8c1d15996e30278
09:14:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.071020 31097 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
09:14:22.095 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.096 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.096 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.096 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.099 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.155 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.156 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.157 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.157 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.159 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.160 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=184), Traces: [0ms] sending RPC to server master-127.29.135.188:44461, [2ms] received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [20ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [40ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [65ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [74ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [74ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [81ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [84ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [94ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [123ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [182ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [184ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, deferred=Deferred@100576516(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1486068290@45d04dc2 -> (continuation of Deferred@1203935816 after retry RPC after error@2048010455) -> (continuation of Deferred@175385628 after retry RPC after error@1292750541) -> (continuation of Deferred@508435966 after retry RPC after error@1970391664) -> (continuation of Deferred@1590561806 after retry RPC after error@2098649788) -> (continuation of Deferred@324634893 after retry RPC after error@1744888668), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/879066594@46803c2e -> (continuation of Deferred@1203935816 after retry RPC after error@2048010455) -> (continuation of Deferred@175385628 after retry RPC after error@1292750541) -> (continuation of Deferred@508435966 after retry RPC after error@1970391664) -> (continuation of Deferred@1590561806 after retry RPC after error@2098649788) -> (continuation of Deferred@324634893 after retry RPC after error@1744888668)))
09:14:22.161 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=184), Traces: [0ms] sending RPC to server master-127.29.135.188:44461, [2ms] received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [20ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [40ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [61ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [65ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [74ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [74ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [79ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [81ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [84ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [93ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [94ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [121ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [123ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [182ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [184ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [184ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, deferred=Deferred@100576516(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$117/1486068290@45d04dc2 -> (continuation of Deferred@1203935816 after retry RPC after error@2048010455) -> (continuation of Deferred@175385628 after retry RPC after error@1292750541) -> (continuation of Deferred@508435966 after retry RPC after error@1970391664) -> (continuation of Deferred@1590561806 after retry RPC after error@2098649788) -> (continuation of Deferred@324634893 after retry RPC after error@1744888668), errback=org.apache.kudu.client.KuduTransaction$$Lambda$118/879066594@46803c2e -> (continuation of Deferred@1203935816 after retry RPC after error@2048010455) -> (continuation of Deferred@175385628 after retry RPC after error@1292750541) -> (continuation of Deferred@508435966 after retry RPC after error@1970391664) -> (continuation of Deferred@1590561806 after retry RPC after error@2098649788) -> (continuation of Deferred@324634893 after retry RPC after error@1744888668)))
09:14:22.174 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.175 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.175 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.177 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.178 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.195 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.196 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.196 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.197 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.203 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.215 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.216 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.216 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.217 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.222 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.235 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.236 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.236 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.237 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.239 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.274 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.275 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.276 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.276 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.277 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.295 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.296 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.296 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.297 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.299 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.315 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.316 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.316 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.318 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.335 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.336 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.336 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.336 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.338 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.395 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.396 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.397 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.397 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.398 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.399 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [1ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [3ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [26ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [29ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [46ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [64ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [102ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [124ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [142ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [144ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [162ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [163ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [223ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [224ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, deferred=Deferred@2096402684(state=PENDING, result=null, callback=(continuation of Deferred@336049548 after retry RPC after error@1413373578) -> (continuation of Deferred@488441606 after retry RPC after error@1169557289) -> (continuation of Deferred@1815372230 after retry RPC after error@1255717060) -> (continuation of Deferred@1057534894 after retry RPC after error@1257403433) -> (continuation of Deferred@92739296 after retry RPC after error@177639436) -> (continuation of Deferred@1887029043 after retry RPC after error@703833641) -> (continuation of Deferred@352228833 after retry RPC after error@710304130) -> (continuation of Deferred@346586020 after retry RPC after error@543049206), errback=(continuation of Deferred@336049548 after retry RPC after error@1413373578) -> (continuation of Deferred@488441606 after retry RPC after error@1169557289) -> (continuation of Deferred@1815372230 after retry RPC after error@1255717060) -> (continuation of Deferred@1057534894 after retry RPC after error@1257403433) -> (continuation of Deferred@92739296 after retry RPC after error@177639436) -> (continuation of Deferred@1887029043 after retry RPC after error@703833641) -> (continuation of Deferred@352228833 after retry RPC after error@710304130) -> (continuation of Deferred@346586020 after retry RPC after error@543049206)))
09:14:22.400 [DEBUG - kudu-nio-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=224), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [1ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [3ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [4ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [4ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [22ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [26ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [29ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [29ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [43ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [46ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [62ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [64ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [65ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [66ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [102ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [103ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [122ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [124ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [124ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [142ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [144ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [144ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [162ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [163ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [164ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [223ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [224ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [224ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [224ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, deferred=Deferred@2096402684(state=PENDING, result=null, callback=(continuation of Deferred@336049548 after retry RPC after error@1413373578) -> (continuation of Deferred@488441606 after retry RPC after error@1169557289) -> (continuation of Deferred@1815372230 after retry RPC after error@1255717060) -> (continuation of Deferred@1057534894 after retry RPC after error@1257403433) -> (continuation of Deferred@92739296 after retry RPC after error@177639436) -> (continuation of Deferred@1887029043 after retry RPC after error@703833641) -> (continuation of Deferred@352228833 after retry RPC after error@710304130) -> (continuation of Deferred@346586020 after retry RPC after error@543049206), errback=(continuation of Deferred@336049548 after retry RPC after error@1413373578) -> (continuation of Deferred@488441606 after retry RPC after error@1169557289) -> (continuation of Deferred@1815372230 after retry RPC after error@1255717060) -> (continuation of Deferred@1057534894 after retry RPC after error@1257403433) -> (continuation of Deferred@92739296 after retry RPC after error@177639436) -> (continuation of Deferred@1887029043 after retry RPC after error@703833641) -> (continuation of Deferred@352228833 after retry RPC after error@710304130) -> (continuation of Deferred@346586020 after retry RPC after error@543049206)))
09:14:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.405519 31097 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:14:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.406000 31097 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:22.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.408075 31097 leader_election.cc:290] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 317bacb8d32f45ee92c35da517f50844 (127.29.135.189:35713), 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461)
09:14:22.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.408780 30377 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "317bacb8d32f45ee92c35da517f50844" is_pre_election: true
09:14:22.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.409399 30377 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4f1ff44df6544f6ab375ae8c1fa999f6 in term 2.
09:14:22.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.410279 30269 leader_election.cc:304] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 317bacb8d32f45ee92c35da517f50844, 4f1ff44df6544f6ab375ae8c1fa999f6; no voters:
09:14:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.410892 31097 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Leader pre-election won for term 3
09:14:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.411186 31097 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:14:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.411477 31097 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 2 FOLLOWER]: Advancing to term 3
09:14:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.411628 30267 leader_election.cc:336] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.414 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.415 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.415 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.415 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.418099 31097 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:22.420 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.420181 30377 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" candidate_term: 3 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "317bacb8d32f45ee92c35da517f50844"
09:14:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.420682 30377 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 2 FOLLOWER]: Advancing to term 3
09:14:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.421046 31097 leader_election.cc:290] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 3 election: Requested vote from peers 317bacb8d32f45ee92c35da517f50844 (127.29.135.189:35713), 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461)
09:14:22.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.423418 30267 leader_election.cc:336] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.426165 30377 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4f1ff44df6544f6ab375ae8c1fa999f6 in term 3.
09:14:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.426947 30269 leader_election.cc:304] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 317bacb8d32f45ee92c35da517f50844, 4f1ff44df6544f6ab375ae8c1fa999f6; no voters: 7f85076611264064a8c1d15996e30278
09:14:22.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.427570 31097 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 3 FOLLOWER]: Leader election won for term 3
09:14:22.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.428084 31097 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [term 3 LEADER]: Becoming Leader. State: Replica: 4f1ff44df6544f6ab375ae8c1fa999f6, State: Running, Role: LEADER
09:14:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.429138 31097 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } }
09:14:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.434492 31101 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4f1ff44df6544f6ab375ae8c1fa999f6. Latest consensus state: current_term: 3 leader_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:22.435 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.435041 31101 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: This master's current role is: LEADER
09:14:22.437 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.439338 31105 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
09:14:22.439 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.446892 31105 catalog_manager.cc:679] Loaded metadata for table txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78]
09:14:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.448566 31105 catalog_manager.cc:679] Loaded metadata for table kudu_system.kudu_transactions [id=713eb997bab6492f9beb8cb357fd218d]
09:14:22.454 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.455302 31105 tablet_loader.cc:96] loaded metadata for tablet 09fb3f24486042db8e27a473de715973 (table txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78])
09:14:22.456 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.456 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.456 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.457197 31105 tablet_loader.cc:96] loaded metadata for tablet 30527506731f4ccb9dd46fbef2e4136e (table txn_manager_fallback [id=467c81dd39584ccb95d4ec6a6d4c0d78])
09:14:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.458608 31105 tablet_loader.cc:96] loaded metadata for tablet 340913df5e7a4d35bab2c47335b2fc17 (table kudu_system.kudu_transactions [id=713eb997bab6492f9beb8cb357fd218d])
09:14:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.459949 31105 catalog_manager.cc:1494] Initializing Kudu cluster ID...
09:14:22.462 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.464083 31105 catalog_manager.cc:1269] Loaded cluster ID: b4bbe5694ba54b4bb19b829763a5a65f
09:14:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.464347 31105 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
09:14:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.469309 31105 catalog_manager.cc:1514] Loading token signing keys...
09:14:22.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.473502 31105 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6: Loaded TSK: 0
09:14:22.474 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.474799 31105 catalog_manager.cc:1524] Initializing in-progress tserver states...
09:14:22.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.481 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627; will retry
09:14:22.494 [DEBUG - pool-38-thread-1] (Connection.java:748) [peer master-127.29.135.188:44461(127.29.135.188:44461)] connecting to peer
09:14:22.496 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.496 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.188:44461(127.29.135.188:44461)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.496 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.135.188:44461: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461
09:14:22.504 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:14:22.505 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=5, TimeoutTracker(timeout=250, elapsed=91), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [1ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [3ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [23ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [24ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [46ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [48ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [63ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [66ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [67ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [89ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [90ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, deferred=Deferred@1226136865(state=PENDING, result=null, callback=(continuation of Deferred@651369259 after retry RPC after error@58583888) -> (continuation of Deferred@769507742 after retry RPC after error@1312980305) -> (continuation of Deferred@1886793201 after retry RPC after error@482181060) -> (continuation of Deferred@1596955900 after retry RPC after error@39504573), errback=(continuation of Deferred@651369259 after retry RPC after error@58583888) -> (continuation of Deferred@769507742 after retry RPC after error@1312980305) -> (continuation of Deferred@1886793201 after retry RPC after error@482181060) -> (continuation of Deferred@1596955900 after retry RPC after error@39504573))) after lookup
09:14:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.521169 30377 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [term 3 FOLLOWER]: Refusing update from remote peer 4f1ff44df6544f6ab375ae8c1fa999f6: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 12. (index mismatch)
09:14:22.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.523317 31101 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.001s
09:14:22.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.543969 30267 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 -> Peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Couldn't send request to peer 7f85076611264064a8c1d15996e30278. Status: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:14:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.553135 31096 sys_catalog.cc:455] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4f1ff44df6544f6ab375ae8c1fa999f6. Latest consensus state: current_term: 3 leader_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.554001 31096 sys_catalog.cc:458] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: This master's current role is: FOLLOWER
09:14:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.569160 31097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:22.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.569977 31097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 [sys.catalog]: This master's current role is: LEADER
09:14:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.601501 31010 heartbeater.cc:646] Failed to heartbeat to 127.29.135.188:44461 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.621030 31096 sys_catalog.cc:455] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f1ff44df6544f6ab375ae8c1fa999f6" member_type: VOTER last_known_addr { host: "127.29.135.190" port: 41627 } } peers { permanent_uuid: "317bacb8d32f45ee92c35da517f50844" member_type: VOTER last_known_addr { host: "127.29.135.189" port: 35713 } } peers { permanent_uuid: "7f85076611264064a8c1d15996e30278" member_type: VOTER last_known_addr { host: "127.29.135.188" port: 44461 } } }
09:14:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.621760 31096 sys_catalog.cc:458] T 00000000000000000000000000000000 P 317bacb8d32f45ee92c35da517f50844 [sys.catalog]: This master's current role is: FOLLOWER
09:14:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.623835 30324 debug-util.cc:398] Leaking SignalData structure 0x7b080007a140 after lost signal to thread 30251
09:14:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.625365 30863 heartbeater.cc:646] Failed to heartbeat to 127.29.135.188:44461 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.640594 30746 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.094s from last keepalive heartbeat (effective timeout is 1.000s)
09:14:22.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.668046 30751 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.29.135.188:44461)
09:14:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.670979 30313 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40238 (request call id 21) took 159 ms (client timeout 159 ms). Trace:
09:14:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.671368 30313 rpcz_store.cc:269] 1029 09:14:22.506775 (+ 0us) service_pool.cc:168] Inserting onto call queue
09:14:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:22.506914 (+ 139us) service_pool.cc:225] Handling call
09:14:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 09:14:22.666467 (+159553us) inbound_call.cc:173] Queueing success response
09:14:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":1164,"init.queue_time_us":11219,"thread_start_us":48777,"threads_started":5}
09:14:22.673 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=258), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [1ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [3ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [23ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [24ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [46ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [48ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [63ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [66ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [67ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [89ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [90ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [92ms] sending RPC to server master-127.29.135.190:41627, [259ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1226136865(state=PENDING, result=null, callback=(continuation of Deferred@651369259 after retry RPC after error@58583888) -> (continuation of Deferred@769507742 after retry RPC after error@1312980305) -> (continuation of Deferred@1886793201 after retry RPC after error@482181060) -> (continuation of Deferred@1596955900 after retry RPC after error@39504573) -> (continuation of Deferred@1124230818 after retry RPC@884367961), errback=(continuation of Deferred@651369259 after retry RPC after error@58583888) -> (continuation of Deferred@769507742 after retry RPC after error@1312980305) -> (continuation of Deferred@1886793201 after retry RPC after error@482181060) -> (continuation of Deferred@1596955900 after retry RPC after error@39504573) -> (continuation of Deferred@1124230818 after retry RPC@884367961)))
09:14:22.674 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=6, TimeoutTracker(timeout=250, elapsed=258), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [1ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [3ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [23ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [24ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [25ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [25ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [42ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [46ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [48ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [63ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [64ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [66ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [67ms] delaying RPC due to: Service unavailable: Master config (127.29.135.188:44461,127.29.135.189:35713,127.29.135.190:41627) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.188:44461, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.189:35713, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.135.190:41627, [82ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.188:44461: Network error: Failed to connect to peer master-127.29.135.188:44461(127.29.135.188:44461): Connection refused: /127.29.135.188:44461, [89ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.189:35713: OK, [90ms] Sub RPC ConnectToMaster: received response from server master-127.29.135.190:41627: OK, [92ms] sending RPC to server master-127.29.135.190:41627, [259ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1226136865(state=PENDING, result=null, callback=(continuation of Deferred@651369259 after retry RPC after error@58583888) -> (continuation of Deferred@769507742 after retry RPC after error@1312980305) -> (continuation of Deferred@1886793201 after retry RPC after error@482181060) -> (continuation of Deferred@1596955900 after retry RPC after error@39504573) -> (continuation of Deferred@1124230818 after retry RPC@884367961), errback=(continuation of Deferred@651369259 after retry RPC after error@58583888) -> (continuation of Deferred@769507742 after retry RPC after error@1312980305) -> (continuation of Deferred@1886793201 after retry RPC after error@482181060) -> (continuation of Deferred@1596955900 after retry RPC after error@39504573) -> (continuation of Deferred@1124230818 after retry RPC@884367961)))
09:14:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.693081 30714 heartbeater.cc:646] Failed to heartbeat to 127.29.135.188:44461 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111)
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:22.637130 30282 catalog_manager.cc:2257] Servicing CreateTable request from {username='slave'} at 127.0.0.1:52334:
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
09:14:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
09:14:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
09:14:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.705299 30282 catalog_manager.cc:7011] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
09:14:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:22.718830 31121 tablet_replica.cc:1306] Aborted: operation has been aborted: cancelling pending write operations
09:14:22.800 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1761729261 last_transition_timestamp: 1761729262
09:14:23.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:23.086637 30267 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 -> Peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Couldn't send request to peer 7f85076611264064a8c1d15996e30278. Status: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
09:14:23.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.595436 31012 heartbeater.cc:499] Master 127.29.135.190:41627 was elected leader, sending a full tablet report...
09:14:23.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:23.612345 30267 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4f1ff44df6544f6ab375ae8c1fa999f6 -> Peer 7f85076611264064a8c1d15996e30278 (127.29.135.188:44461): Couldn't send request to peer 7f85076611264064a8c1d15996e30278. Status: Network error: Client connection negotiation failed: client connection to 127.29.135.188:44461: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
09:14:23.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.626255 30865 heartbeater.cc:499] Master 127.29.135.190:41627 was elected leader, sending a full tablet report...
09:14:23.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.678357 30717 heartbeater.cc:499] Master 127.29.135.190:41627 was elected leader, sending a full tablet report...
09:14:23.809 [DEBUG - main] (Connection.java:575) [peer master-127.29.135.189:35713(127.29.135.189:35713)] disconnecting while in state READY
09:14:23.810 [DEBUG - main] (Connection.java:575) [peer c9d202cd45da486cbe20e2093811658e(127.29.135.129:36359)] disconnecting while in state READY
09:14:23.810 [DEBUG - main] (Connection.java:575) [peer master-127.29.135.190:41627(127.29.135.190:41627)] disconnecting while in state READY
09:14:23.810 [DEBUG - main] (Connection.java:575) [peer master-127.29.135.188:44461(127.29.135.188:44461)] disconnecting while in state TERMINATED
09:14:23.810 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.29.135.189:35713(127.29.135.189:35713)] handling channelInactive
09:14:23.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.135.189:35713(127.29.135.189:35713)] cleaning up while in state READY due to: connection closed
09:14:23.811 [DEBUG - kudu-nio-1] (Connection.java:239) [peer c9d202cd45da486cbe20e2093811658e(127.29.135.129:36359)] handling channelInactive
09:14:23.811 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:14:23.811 [DEBUG - kudu-nio-1] (Connection.java:709) [peer c9d202cd45da486cbe20e2093811658e(127.29.135.129:36359)] cleaning up while in state READY due to: connection closed
09:14:23.813 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.135.190:41627(127.29.135.190:41627)] handling channelInactive
09:14:23.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.135.190:41627(127.29.135.190:41627)] cleaning up while in state READY due to: connection closed
09:14:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.814105 30238 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu with pid 30570
09:14:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:23.841606 30738 connection.cc:537] client connection to 127.29.135.129:36359 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:14:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 09:14:23.842072 30738 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
09:14:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.842621 30238 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu with pid 30719
09:14:23.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.880666 30238 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu with pid 30868
09:14:23.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.912051 30238 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu with pid 30250
09:14:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 09:14:23.948293 30238 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task8A6J2s/build/tsan/bin/kudu with pid 30330
09:14:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T09:14:25Z chronyd exiting
09:14:26.158 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1761729261 last_transition_timestamp: 1761729262
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537) ~[main/:?]
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284) ~[main/:?]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Failed to write to server: 2ea4ee372f2f44c9878debc9a6ba4bf7 (127.29.135.130:40585): transaction ID 0 is not open: state: ABORTED user: "slave" start_timestamp: 1761729261 last_transition_timestamp: 1761729262
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduTransaction.doCommitTransaction(KuduTransaction.java:491)
at org.apache.kudu.client.KuduTransaction.commitWithMode(KuduTransaction.java:537)
at org.apache.kudu.client.KuduTransaction.commit(KuduTransaction.java:284)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveSwitchesToOtherTxnManager(TestKuduTransaction.java:1152)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.RpcProxy.dispatchTxnManagerError(RpcProxy.java:439)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:298)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more