14:13:13.168 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:13:13.169 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNSAC30/test-tmp
14:13:13.169 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:13:13.169 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNSAC30/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:13:13.170 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:13:13.493 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:13:13.493 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:13:13.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.498310 19025 mini_oidc.cc:150] Starting JWKS server
14:13:13.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.502784 19025 webserver.cc:469] Webserver started at http://127.0.0.1:36457/ using document root <none> and password file <none>
14:13:13.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.503358 19025 mini_oidc.cc:186] Starting OIDC Discovery server
14:13:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.505594 19025 webserver.cc:469] Webserver started at http://127.0.0.1:32871/ using document root <none> and password file <none>
14:13:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-22T14:13:13Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:13:13.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-22T14:13:13Z Disabled control of system clock
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.572799 19025 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/logs
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/info.pb
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.126:46297
14:13:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.126
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:13:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:13:13.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.873953 19037 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:13.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.874485 19037 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:13.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.875080 19037 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.907970 19037 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.908278 19037 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:13:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.908457 19037 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.908654 19037 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.908912 19037 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.909116 19037 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.909258 19037 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36457/jwks.json
14:13:13.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.947517 19037 master_runner.cc:386] Master server non-default flags:
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.126:46297
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/info.pb
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.126
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/logs
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6717
14:13:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:13.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.948938 19037 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:13.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.950615 19037 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.967130 19044 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.967594 19043 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:13.967692 19046 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:13.968811 19037 server_base.cc:1048] running on GCE node
14:13:15.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.149058 19037 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.152299 19037 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.153755 19037 hybrid_clock.cc:648] HybridClock initialized: now 1750601595153684 us; error 99 us; skew 500 ppm
14:13:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.154577 19037 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.165009 19037 webserver.cc:469] Webserver started at http://127.18.148.126:35611/ using document root <none> and password file <none>
14:13:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.165967 19037 fs_manager.cc:362] Metadata directory not provided
14:13:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.166162 19037 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.166564 19037 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:13:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.171041 19037 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/instance:
14:13:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8cc3150135844c45b7060ca819868a7b"
14:13:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:15 on dist-test-slave-p5xv"
14:13:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.172115 19037 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal/instance:
14:13:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8cc3150135844c45b7060ca819868a7b"
14:13:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:15 on dist-test-slave-p5xv"
14:13:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.179425 19037 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
14:13:15.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.185472 19053 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:15.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.186527 19037 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.186832 19037 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data,/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8cc3150135844c45b7060ca819868a7b"
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:15 on dist-test-slave-p5xv"
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.187139 19037 fs_report.cc:389] FS layout report
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/data
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.240473 19037 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.242010 19037 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.242417 19037 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:15.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.326715 19037 rpc_server.cc:307] RPC server started. Bound to: 127.18.148.126:46297
14:13:15.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.326820 19114 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.148.126:46297 every 8 connection(s)
14:13:15.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.329556 19037 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/info.pb
14:13:15.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.332928 19025 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu as pid 19037
14:13:15.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.333490 19025 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal/instance
14:13:15.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.335673 19115 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.338822 19025 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/wal
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/logs
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data/info.pb
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.125:40291
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.125
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:13:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:13:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.350078 19115 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } has no permanent_uuid. Determining permanent_uuid...
14:13:15.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.377629 19115 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } has no permanent_uuid. Determining permanent_uuid...
14:13:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.379977 19054 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:15.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.384434 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.125:40291: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:15.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.433949 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } attempt: 1
14:13:15.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.438412 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.125:40291: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:15.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.506841 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } attempt: 2
14:13:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.511209 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.125:40291: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.602607 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } attempt: 3
14:13:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.606789 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.125:40291: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.653645 19117 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.654312 19117 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.654896 19117 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.688127 19117 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.688477 19117 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:13:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.688769 19117 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.689092 19117 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.689369 19117 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.689646 19117 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.689855 19117 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36457/jwks.json
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.725312 19117 master_runner.cc:386] Master server non-default flags:
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data
14:13:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/wal
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.125:40291
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data/info.pb
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.125
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/logs
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:13:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6717
14:13:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.727056 19117 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:15.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.728772 19117 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:15.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.742261 19126 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:15.750170 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } attempt: 4
14:13:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.755801 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.125:40291: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:16.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.055285 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } attempt: 5
14:13:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:16.062117 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.125:40291: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:16.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.609802 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } attempt: 6
14:13:16.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:16.620278 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.125:40291: Network error: Client connection negotiation failed: client connection to 127.18.148.125:40291: connect: Connection refused (error 111)
14:13:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:15.742682 19127 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:16.939851 19129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:16.942049 19128 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1196 milliseconds
14:13:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.942183 19117 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:13:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.943475 19117 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:16.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.946370 19117 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:16.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.947829 19117 hybrid_clock.cc:648] HybridClock initialized: now 1750601596947779 us; error 54 us; skew 500 ppm
14:13:16.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.948755 19117 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:16.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.955715 19117 webserver.cc:469] Webserver started at http://127.18.148.125:45167/ using document root <none> and password file <none>
14:13:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.956684 19117 fs_manager.cc:362] Metadata directory not provided
14:13:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.956948 19117 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:16.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.957471 19117 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:13:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.962049 19117 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data/instance:
14:13:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6536e6eac2a047b48c314cf1d86d1db2"
14:13:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:16 on dist-test-slave-p5xv"
14:13:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.963361 19117 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/wal/instance:
14:13:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6536e6eac2a047b48c314cf1d86d1db2"
14:13:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:16 on dist-test-slave-p5xv"
14:13:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.970667 19117 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
14:13:16.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.976657 19138 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.977763 19117 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:13:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.978109 19117 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data,/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/wal
14:13:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6536e6eac2a047b48c314cf1d86d1db2"
14:13:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:16 on dist-test-slave-p5xv"
14:13:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:16.978446 19117 fs_report.cc:389] FS layout report
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/wal
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/wal
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data/data
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.034574 19117 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:17.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.036105 19117 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:17.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.036578 19117 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:17.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.119381 19117 rpc_server.cc:307] RPC server started. Bound to: 127.18.148.125:40291
14:13:17.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.119448 19199 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.148.125:40291 every 8 connection(s)
14:13:17.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.122367 19117 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/data/info.pb
14:13:17.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.126384 19025 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu as pid 19117
14:13:17.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.126881 19025 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-1/wal/instance
14:13:17.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.128219 19200 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.129956 19025 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/wal
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/logs
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data/info.pb
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.124:37047
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.124
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:13:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:13:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.142264 19200 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } has no permanent_uuid. Determining permanent_uuid...
14:13:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.164819 19200 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } has no permanent_uuid. Determining permanent_uuid...
14:13:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.179093 19200 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } has no permanent_uuid. Determining permanent_uuid...
14:13:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.181823 19141 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.185966 19200 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.235442 19200 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 1
14:13:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.239450 19200 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.307837 19200 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 2
14:13:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.311946 19200 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.403368 19200 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 3
14:13:17.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.408087 19200 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.440078 19202 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.440661 19202 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.441174 19202 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.472980 19202 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.473284 19202 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:13:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.473454 19202 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.473650 19202 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.473846 19202 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.474033 19202 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.474177 19202 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36457/jwks.json
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.517700 19202 master_runner.cc:386] Master server non-default flags:
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/wal
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.124:37047
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data/info.pb
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.124
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/logs
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6717
14:13:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.519066 19202 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:17.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.520805 19202 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.534971 19213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:17.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.551559 19200 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 4
14:13:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.557548 19200 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.680896 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } attempt: 7
14:13:17.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.712071 19115 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } has no permanent_uuid. Determining permanent_uuid...
14:13:17.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.721293 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.779891 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 1
14:13:17.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.787002 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.857103 19200 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 5
14:13:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.863837 19200 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.868693 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 2
14:13:17.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.875339 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:17.960944 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 3
14:13:17.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.968142 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:18.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.108842 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 4
14:13:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:18.115808 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:18.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.399633 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 5
14:13:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:18.406598 19115 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.411432 19200 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 6
14:13:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:18.424278 19200 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:18.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.536767 19215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:18.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:17.534991 19212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:18.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:18.770016 19214 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1232 milliseconds
14:13:18.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.770167 19202 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:13:18.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.771360 19202 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.774454 19202 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.775895 19202 hybrid_clock.cc:648] HybridClock initialized: now 1750601598775849 us; error 63 us; skew 500 ppm
14:13:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.776686 19202 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.783532 19202 webserver.cc:469] Webserver started at http://127.18.148.124:38047/ using document root <none> and password file <none>
14:13:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.784442 19202 fs_manager.cc:362] Metadata directory not provided
14:13:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.784636 19202 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.785104 19202 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:13:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.789680 19202 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data/instance:
14:13:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bae83f02308f42caaaad05e349e4b26a"
14:13:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:18 on dist-test-slave-p5xv"
14:13:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.790741 19202 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/wal/instance:
14:13:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bae83f02308f42caaaad05e349e4b26a"
14:13:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:18 on dist-test-slave-p5xv"
14:13:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.797914 19202 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
14:13:18.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.803395 19227 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:18.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.804443 19202 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.804780 19202 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data,/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/wal
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bae83f02308f42caaaad05e349e4b26a"
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:18 on dist-test-slave-p5xv"
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.805097 19202 fs_report.cc:389] FS layout report
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/wal
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/wal
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data/data
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:18.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:18.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.859206 19202 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:18.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.860627 19202 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:18.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.861043 19202 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.944162 19202 rpc_server.cc:307] RPC server started. Bound to: 127.18.148.124:37047
14:13:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.944253 19288 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.148.124:37047 every 8 connection(s)
14:13:18.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.946888 19202 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/data/info.pb
14:13:18.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.951864 19025 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu as pid 19202
14:13:18.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.951951 19289 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:18.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.952208 19025 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-2/wal/instance
14:13:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.959192 19115 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 6
14:13:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.980242 19289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } has no permanent_uuid. Determining permanent_uuid...
14:13:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.998389 19115 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Bootstrap starting.
14:13:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:18.998852 19289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } has no permanent_uuid. Determining permanent_uuid...
14:13:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.004439 19115 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Neither blocks nor log segments found. Creating new log.
14:13:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.006881 19115 log.cc:826] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Log is configured to *not* fsync() on all Append() calls
14:13:19.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.010704 19289 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } has no permanent_uuid. Determining permanent_uuid...
14:13:19.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.012269 19115 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: No bootstrap required, opened a new log
14:13:19.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.028481 19289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: Bootstrap starting.
14:13:19.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.034584 19289 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: Neither blocks nor log segments found. Creating new log.
14:13:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.035183 19115 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.035861 19115 raft_consensus.cc:383] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.036065 19115 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cc3150135844c45b7060ca819868a7b, State: Initialized, Role: FOLLOWER
14:13:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.036959 19289 log.cc:826] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: Log is configured to *not* fsync() on all Append() calls
14:13:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.036690 19115 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.039415 19299 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.040071 19299 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: FOLLOWER
14:13:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.041085 19115 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: configured and running, proceeding with master startup.
14:13:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.042055 19289 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: No bootstrap required, opened a new log
14:13:19.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.064577 19289 raft_consensus.cc:357] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.065299 19289 raft_consensus.cc:383] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:19.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.065797 19289 raft_consensus.cc:738] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bae83f02308f42caaaad05e349e4b26a, State: Initialized, Role: FOLLOWER
14:13:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.066896 19299 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.066882 19289 consensus_queue.cc:260] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.067384 19299 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.081197 19299 leader_election.cc:290] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6536e6eac2a047b48c314cf1d86d1db2 (127.18.148.125:40291), bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047)
14:13:19.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.083662 19312 sys_catalog.cc:455] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.085355 19312 sys_catalog.cc:458] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: This master's current role is: FOLLOWER
14:13:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.080247 19311 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:13:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.086269 19311 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:13:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.087085 19289 sys_catalog.cc:564] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: configured and running, proceeding with master startup.
14:13:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.087332 19254 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8cc3150135844c45b7060ca819868a7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bae83f02308f42caaaad05e349e4b26a" is_pre_election: true
14:13:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.088215 19254 raft_consensus.cc:2466] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cc3150135844c45b7060ca819868a7b in term 0.
14:13:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.093230 19056 leader_election.cc:304] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cc3150135844c45b7060ca819868a7b, bae83f02308f42caaaad05e349e4b26a; no voters:
14:13:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.094506 19299 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Leader pre-election won for term 1
14:13:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.095019 19299 raft_consensus.cc:491] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:13:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.095450 19299 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 0 FOLLOWER]: Advancing to term 1
14:13:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.106169 19299 raft_consensus.cc:513] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.114535 19299 leader_election.cc:290] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [CANDIDATE]: Term 1 election: Requested vote from peers 6536e6eac2a047b48c314cf1d86d1db2 (127.18.148.125:40291), bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047)
14:13:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.115111 19254 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8cc3150135844c45b7060ca819868a7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bae83f02308f42caaaad05e349e4b26a"
14:13:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.115717 19254 raft_consensus.cc:3058] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 0 FOLLOWER]: Advancing to term 1
14:13:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.115981 19233 tablet.cc:2378] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:13:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.081866 19037 master_runner.cc:186] Error getting master registration for 127.18.148.125:40291: OK, instance_id { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" instance_seqno: 1750601597101047 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:13:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.123214 19254 raft_consensus.cc:2466] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cc3150135844c45b7060ca819868a7b in term 1.
14:13:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.124063 19037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.124506 19056 leader_election.cc:304] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cc3150135844c45b7060ca819868a7b, bae83f02308f42caaaad05e349e4b26a; no voters:
14:13:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.125545 19299 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 1 FOLLOWER]: Leader election won for term 1
14:13:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.134536 19299 raft_consensus.cc:695] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 1 LEADER]: Becoming Leader. State: Replica: 8cc3150135844c45b7060ca819868a7b, State: Running, Role: LEADER
14:13:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.135838 19299 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.154549 19316 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8cc3150135844c45b7060ca819868a7b. Latest consensus state: current_term: 1 leader_uuid: "8cc3150135844c45b7060ca819868a7b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.155701 19316 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: LEADER
14:13:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.160462 19321 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:13:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.169054 19321 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:13:19.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.194264 19254 raft_consensus.cc:1273] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 1 FOLLOWER]: Refusing update from remote peer 8cc3150135844c45b7060ca819868a7b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:13:19.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.196262 19316 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [LEADER]: Connected to new peer: Peer: permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:13:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.233635 19312 sys_catalog.cc:455] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8cc3150135844c45b7060ca819868a7b. Latest consensus state: current_term: 1 leader_uuid: "8cc3150135844c45b7060ca819868a7b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.238754 19323 mvcc.cc:204] Tried to move back new op lower bound from 7170464150279200768 to 7170464150073360384. Current Snapshot: MvccSnapshot[applied={T|T < 7170464150279200768}]
14:13:19.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.234614 19312 sys_catalog.cc:458] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: This master's current role is: FOLLOWER
14:13:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.251128 19312 sys_catalog.cc:455] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8cc3150135844c45b7060ca819868a7b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.252135 19312 sys_catalog.cc:458] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: This master's current role is: FOLLOWER
14:13:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.252892 19299 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8cc3150135844c45b7060ca819868a7b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.253831 19299 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: LEADER
14:13:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.260257 19321 catalog_manager.cc:1349] Generated new cluster ID: f5b6242f23d14f4e955c3f8ab9631ca3
14:13:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.260524 19321 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:13:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.275713 19332 catalog_manager.cc:1261] Loaded cluster ID: f5b6242f23d14f4e955c3f8ab9631ca3
14:13:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.276006 19332 catalog_manager.cc:1554] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: loading cluster ID for follower catalog manager: success
14:13:19.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.280179 19332 catalog_manager.cc:1578] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: acquiring CA information for follower catalog manager: Not found: root CA entry not found
14:13:19.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.280519 19332 catalog_manager.cc:875] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
14:13:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.312479 19321 catalog_manager.cc:1372] Generated new certificate authority record
14:13:19.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.314690 19321 catalog_manager.cc:1506] Loading token signing keys...
14:13:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.279417 19202 master_runner.cc:186] Error getting master registration for 127.18.148.125:40291: OK, instance_id { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" instance_seqno: 1750601597101047 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:13:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.316313 19202 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.361557 19321 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Generated new TSK 0
14:13:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.362879 19321 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.401360 19025 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/wal
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/logs
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data/info.pb
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.65:0
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.148.65
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.65
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:13:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:13:19.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.484956 19200 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } attempt: 7
14:13:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.527623 19200 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: Bootstrap starting.
14:13:19.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.550678 19200 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: Neither blocks nor log segments found. Creating new log.
14:13:19.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.564209 19200 log.cc:826] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: Log is configured to *not* fsync() on all Append() calls
14:13:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.606566 19200 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: No bootstrap required, opened a new log
14:13:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.765144 19200 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.770206 19200 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.772229 19200 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6536e6eac2a047b48c314cf1d86d1db2, State: Initialized, Role: FOLLOWER
14:13:19.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.777452 19200 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.823614 19165 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8cc3150135844c45b7060ca819868a7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6536e6eac2a047b48c314cf1d86d1db2" is_pre_election: true
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.824537 19165 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cc3150135844c45b7060ca819868a7b in term 0.
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.825135 19165 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:48250 (request call id 1) took 749 ms (client timeout 378 ms). Trace:
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.825464 19165 rpcz_store.cc:269] 0622 14:13:19.076074 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.076255 (+ 181us) service_pool.cc:225] Handling call
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.825055 (+748800us) inbound_call.cc:173] Queueing success response
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.826581 19161 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8cc3150135844c45b7060ca819868a7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6536e6eac2a047b48c314cf1d86d1db2"
14:13:19.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.827159 19161 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 0 FOLLOWER]: Advancing to term 1
14:13:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.829593 19162 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8cc3150135844c45b7060ca819868a7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6536e6eac2a047b48c314cf1d86d1db2" is_pre_election: true
14:13:19.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.830544 19162 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:48250 (request call id 5) took 363 ms (client timeout 378 ms). Trace:
14:13:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.830858 19162 rpcz_store.cc:269] 0622 14:13:19.467392 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:13:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.467548 (+ 156us) service_pool.cc:225] Handling call
14:13:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.830473 (+362925us) inbound_call.cc:173] Queueing success response
14:13:19.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:13:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.832705 19164 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8cc3150135844c45b7060ca819868a7b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6536e6eac2a047b48c314cf1d86d1db2"
14:13:19.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.834172 19164 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:48250 (request call id 3) took 720 ms (client timeout 376 ms). Trace:
14:13:19.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.834496 19164 rpcz_store.cc:269] 0622 14:13:19.113684 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:13:19.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.113844 (+ 160us) service_pool.cc:225] Handling call
14:13:19.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.834101 (+720257us) inbound_call.cc:173] Queueing success response
14:13:19.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:13:19.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.841507 19161 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cc3150135844c45b7060ca819868a7b in term 1.
14:13:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.842906 19163 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 1 FOLLOWER]: Refusing update from remote peer 8cc3150135844c45b7060ca819868a7b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:13:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.849187 19161 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:48250 (request call id 6) took 341 ms (client timeout 376 ms). Trace:
14:13:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.849440 19299 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:13:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.850077 19161 rpcz_store.cc:269] 0622 14:13:19.507554 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:13:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.507757 (+ 203us) service_pool.cc:225] Handling call
14:13:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:19.849125 (+341368us) inbound_call.cc:173] Queueing success response
14:13:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:13:19.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.815099 19338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.851796 19338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: This master's current role is: FOLLOWER
14:13:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.850948 19200 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: configured and running, proceeding with master startup.
14:13:19.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.915729 19338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8cc3150135844c45b7060ca819868a7b. Latest consensus state: current_term: 1 leader_uuid: "8cc3150135844c45b7060ca819868a7b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.916435 19338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: This master's current role is: FOLLOWER
14:13:19.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.917726 19344 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8cc3150135844c45b7060ca819868a7b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.918637 19344 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: This master's current role is: FOLLOWER
14:13:19.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.966742 19299 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8cc3150135844c45b7060ca819868a7b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:19.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.967672 19299 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: LEADER
14:13:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:19.979497 19360 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:13:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.997951 19334 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.998617 19334 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:20.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:19.999802 19334 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:20.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.063480 19334 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:20.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.064502 19334 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:13:20.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.064841 19334 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:13:20.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.065099 19334 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:13:20.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.065862 19334 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.148.65
14:13:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:20.132087 19334 tablet_server_runner.cc:78] Tablet server non-default flags:
14:13:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/wal
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.65:0
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data/info.pb
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.65
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.148.65
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/logs
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6717
14:13:20.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:20.133867 19334 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:20.136003 19334 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:20.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.177923 19363 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:20.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:20.294118 19332 catalog_manager.cc:1576] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: acquiring CA information for follower catalog manager: success
14:13:20.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:20.303112 19332 catalog_manager.cc:1604] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:13:20.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.521085 19111 debug-util.cc:398] Leaking SignalData structure 0x7b0800079900 after lost signal to thread 19038
14:13:20.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.522317 19111 debug-util.cc:398] Leaking SignalData structure 0x7b08000795a0 after lost signal to thread 19114
14:13:21.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.001900 19360 catalog_manager.cc:1261] Loaded cluster ID: f5b6242f23d14f4e955c3f8ab9631ca3
14:13:21.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.002193 19360 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: loading cluster ID for follower catalog manager: success
14:13:21.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.017886 19360 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: acquiring CA information for follower catalog manager: success
14:13:21.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.044255 19360 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:13:21.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.193883 19366 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:21.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:21.703142 19364 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1523 milliseconds
14:13:21.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.703346 19334 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:13:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:20.184623 19362 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.715369 19334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:21.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.723798 19334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.725443 19334 hybrid_clock.cc:648] HybridClock initialized: now 1750601601725367 us; error 87 us; skew 500 ppm
14:13:21.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.726658 19334 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:21.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.766542 19334 webserver.cc:469] Webserver started at http://127.18.148.65:44653/ using document root <none> and password file <none>
14:13:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.768424 19334 fs_manager.cc:362] Metadata directory not provided
14:13:21.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.769261 19334 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:21.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.770002 19334 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:13:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.777900 19334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data/instance:
14:13:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef36fff320fc4f2d9f7c375323b74a1b"
14:13:21.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:21 on dist-test-slave-p5xv"
14:13:21.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.788900 19334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/wal/instance:
14:13:21.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef36fff320fc4f2d9f7c375323b74a1b"
14:13:21.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:21 on dist-test-slave-p5xv"
14:13:21.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.814913 19334 fs_manager.cc:696] Time spent creating directory manager: real 0.025s user 0.009s sys 0.001s
14:13:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.839877 19374 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.843026 19334 fs_manager.cc:730] Time spent opening block manager: real 0.019s user 0.006s sys 0.001s
14:13:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.843588 19334 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data,/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/wal
14:13:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef36fff320fc4f2d9f7c375323b74a1b"
14:13:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:21 on dist-test-slave-p5xv"
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:21.844455 19334 fs_report.cc:389] FS layout report
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/wal
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/wal
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data/data
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:22.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.011963 19334 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.025785 19334 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:22.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.026631 19334 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:22.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.110183 19334 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:13:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.110764 19334 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
14:13:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.111258 19334 ts_tablet_manager.cc:610] Registered 0 tablets
14:13:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.111596 19334 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:13:22.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.812791 19334 rpc_server.cc:307] RPC server started. Bound to: 127.18.148.65:44837
14:13:22.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.812938 19499 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.148.65:44837 every 8 connection(s)
14:13:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.816998 19334 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/data/info.pb
14:13:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.827035 19025 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu as pid 19334
14:13:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.827505 19025 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-0/wal/instance
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.841969 19025 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/wal
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/logs
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data/info.pb
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.66:0
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.148.66
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.66
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:22.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:22.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:13:22.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:13:22.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:13:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.891281 19500 heartbeater.cc:344] Connected to a master server at 127.18.148.124:37047
14:13:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.891670 19505 heartbeater.cc:344] Connected to a master server at 127.18.148.126:46297
14:13:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.891788 19500 heartbeater.cc:461] Registering TS with master...
14:13:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.892187 19505 heartbeater.cc:461] Registering TS with master...
14:13:22.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.893349 19500 heartbeater.cc:507] Master 127.18.148.124:37047 requested a full tablet report, sending...
14:13:22.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.893599 19505 heartbeater.cc:507] Master 127.18.148.126:46297 requested a full tablet report, sending...
14:13:22.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.896761 19069 ts_manager.cc:194] Registered new tserver with Master: ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837)
14:13:22.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.897308 19244 ts_manager.cc:194] Registered new tserver with Master: ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837)
14:13:22.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.900139 19069 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.148.65:33915
14:13:22.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.919402 19501 heartbeater.cc:344] Connected to a master server at 127.18.148.125:40291
14:13:22.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.919844 19501 heartbeater.cc:461] Registering TS with master...
14:13:22.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.920564 19501 heartbeater.cc:507] Master 127.18.148.125:40291 requested a full tablet report, sending...
14:13:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:22.923811 19155 ts_manager.cc:194] Registered new tserver with Master: ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837)
14:13:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.278223 19504 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.278788 19504 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.279451 19504 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:23.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.314325 19504 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.315006 19504 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:13:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.315253 19504 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:13:23.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.315500 19504 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:13:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.315974 19504 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.148.66
14:13:23.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:23.354996 19504 tablet_server_runner.cc:78] Tablet server non-default flags:
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/wal
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.66:0
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data/info.pb
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.66
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.148.66
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/logs
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6717
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:23.356509 19504 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:23.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:23.358659 19504 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:23.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.380501 19514 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:23.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:23.904695 19505 heartbeater.cc:499] Master 127.18.148.126:46297 was elected leader, sending a full tablet report...
14:13:25.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:23.381225 19515 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:24.775249 19513 debug-util.cc:398] Leaking SignalData structure 0x7b08000184c0 after lost signal to thread 19504
14:13:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.066913 19504 thread.cc:640] GCE (cloud detector) Time spent creating pthread: real 1.690s user 0.528s sys 0.966s
14:13:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.067273 19504 thread.cc:606] GCE (cloud detector) Time spent starting thread: real 1.691s user 0.528s sys 0.966s
14:13:25.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.075387 19522 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:25.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.075348 19504 server_base.cc:1048] running on GCE node
14:13:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.076642 19504 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.078667 19504 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.079998 19504 hybrid_clock.cc:648] HybridClock initialized: now 1750601605079966 us; error 31 us; skew 500 ppm
14:13:25.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.080806 19504 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:25.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.086869 19504 webserver.cc:469] Webserver started at http://127.18.148.66:44137/ using document root <none> and password file <none>
14:13:25.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.087775 19504 fs_manager.cc:362] Metadata directory not provided
14:13:25.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.087961 19504 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:25.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.088346 19504 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:13:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.092864 19504 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data/instance:
14:13:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a31e618d3e354bdb84eca1dd32db33ec"
14:13:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:25 on dist-test-slave-p5xv"
14:13:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.093961 19504 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/wal/instance:
14:13:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a31e618d3e354bdb84eca1dd32db33ec"
14:13:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:25 on dist-test-slave-p5xv"
14:13:25.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.101585 19504 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.002s
14:13:25.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.107705 19527 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.109198 19504 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
14:13:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.109495 19504 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data,/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/wal
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a31e618d3e354bdb84eca1dd32db33ec"
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:25 on dist-test-slave-p5xv"
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.109812 19504 fs_report.cc:389] FS layout report
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/wal
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/wal
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data/data
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:25.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.180207 19504 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:25.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.181679 19504 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.182111 19504 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.191727 19504 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:13:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.191967 19504 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:13:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.192272 19504 ts_tablet_manager.cc:610] Registered 0 tablets
14:13:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.192449 19504 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:13:25.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.379192 19504 rpc_server.cc:307] RPC server started. Bound to: 127.18.148.66:38233
14:13:25.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.379665 19651 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.148.66:38233 every 8 connection(s)
14:13:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.381891 19504 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/data/info.pb
14:13:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.386165 19025 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu as pid 19504
14:13:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.386919 19025 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-1/wal/instance
14:13:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.396557 19654 heartbeater.cc:344] Connected to a master server at 127.18.148.126:46297
14:13:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.397202 19654 heartbeater.cc:461] Registering TS with master...
14:13:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.401481 19654 heartbeater.cc:507] Master 127.18.148.126:46297 requested a full tablet report, sending...
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.402281 19025 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/wal
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/logs
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data/info.pb
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.67:0
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.148.67
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.67
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:13:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:13:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
14:13:25.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.406466 19652 heartbeater.cc:344] Connected to a master server at 127.18.148.124:37047
14:13:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.407053 19652 heartbeater.cc:461] Registering TS with master...
14:13:25.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.408071 19652 heartbeater.cc:507] Master 127.18.148.124:37047 requested a full tablet report, sending...
14:13:25.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.410904 19244 ts_manager.cc:194] Registered new tserver with Master: a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233)
14:13:25.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.410830 19069 ts_manager.cc:194] Registered new tserver with Master: a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233)
14:13:25.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.412920 19069 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.148.66:53929
14:13:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.417024 19653 heartbeater.cc:344] Connected to a master server at 127.18.148.125:40291
14:13:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.417311 19653 heartbeater.cc:461] Registering TS with master...
14:13:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.417830 19653 heartbeater.cc:507] Master 127.18.148.125:40291 requested a full tablet report, sending...
14:13:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.419763 19155 ts_manager.cc:194] Registered new tserver with Master: a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233)
14:13:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.724063 19657 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.724658 19657 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.725380 19657 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.760969 19657 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.761669 19657 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
14:13:25.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.761910 19657 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
14:13:25.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.762143 19657 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
14:13:25.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.762650 19657 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.18.148.67
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.798875 19657 tablet_server_runner.cc:78] Tablet server non-default flags:
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/wal
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.67:0
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data/info.pb
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.67
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.18.148.67
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/logs
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6717
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.800673 19657 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:25.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:25.802505 19657 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.821183 19664 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:26.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:26.417778 19654 heartbeater.cc:499] Master 127.18.148.126:46297 was elected leader, sending a full tablet report...
14:13:27.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:25.821758 19665 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:27.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:27.215839 19666 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1395 milliseconds
14:13:27.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:27.216873 19667 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:27.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:27.220994 19657 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.404s user 0.446s sys 0.933s
14:13:27.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:27.221612 19657 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.405s user 0.447s sys 0.933s
14:13:27.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.221953 19657 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:13:27.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.223614 19657 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.226593 19657 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:27.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.228106 19657 hybrid_clock.cc:648] HybridClock initialized: now 1750601607228003 us; error 92 us; skew 500 ppm
14:13:27.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.229369 19657 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.242174 19657 webserver.cc:469] Webserver started at http://127.18.148.67:38005/ using document root <none> and password file <none>
14:13:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.243623 19657 fs_manager.cc:362] Metadata directory not provided
14:13:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.243933 19657 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:27.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.244555 19657 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:13:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.252406 19657 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data/instance:
14:13:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f12ed04657c44a34b657c4b771901c09"
14:13:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:27 on dist-test-slave-p5xv"
14:13:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.254206 19657 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/wal/instance:
14:13:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f12ed04657c44a34b657c4b771901c09"
14:13:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:27 on dist-test-slave-p5xv"
14:13:27.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.264855 19657 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.009s sys 0.002s
14:13:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.271346 19675 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:27.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.272655 19657 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.273030 19657 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data,/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/wal
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f12ed04657c44a34b657c4b771901c09"
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:27 on dist-test-slave-p5xv"
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.273355 19657 fs_report.cc:389] FS layout report
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/wal
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/wal
14:13:27.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data/data
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.346403 19657 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.347829 19657 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.348279 19657 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.358603 19657 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:13:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.358860 19657 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:13:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.359169 19657 ts_tablet_manager.cc:610] Registered 0 tablets
14:13:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.359350 19657 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:13:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.566552 19657 rpc_server.cc:307] RPC server started. Bound to: 127.18.148.67:45649
14:13:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.567042 19800 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.148.67:45649 every 8 connection(s)
14:13:27.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.569231 19657 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/data/info.pb
14:13:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.580202 19025 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu as pid 19657
14:13:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.580693 19025 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/ts-2/wal/instance
14:13:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.582163 19803 heartbeater.cc:344] Connected to a master server at 127.18.148.126:46297
14:13:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.582662 19803 heartbeater.cc:461] Registering TS with master...
14:13:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.584093 19803 heartbeater.cc:507] Master 127.18.148.126:46297 requested a full tablet report, sending...
14:13:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.588785 19069 ts_manager.cc:194] Registered new tserver with Master: f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.590663 19069 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.18.148.67:51999
14:13:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.592103 19801 heartbeater.cc:344] Connected to a master server at 127.18.148.124:37047
14:13:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.592492 19801 heartbeater.cc:461] Registering TS with master...
14:13:27.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.593479 19801 heartbeater.cc:507] Master 127.18.148.124:37047 requested a full tablet report, sending...
14:13:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.596442 19802 heartbeater.cc:344] Connected to a master server at 127.18.148.125:40291
14:13:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.596767 19802 heartbeater.cc:461] Registering TS with master...
14:13:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.597287 19243 ts_manager.cc:194] Registered new tserver with Master: f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:27.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.597416 19802 heartbeater.cc:507] Master 127.18.148.125:40291 requested a full tablet report, sending...
14:13:27.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.600144 19155 ts_manager.cc:194] Registered new tserver with Master: f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:27.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.620599 19025 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
14:13:27.632 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:13:27.632 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:13:27.633 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.148.126"
port: 46297
}
bound_http_address {
host: "127.18.148.126"
port: 35611
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.148.125"
port: 40291
}
bound_http_address {
host: "127.18.148.125"
port: 45167
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.148.124"
port: 37047
}
bound_http_address {
host: "127.18.148.124"
port: 38047
}
}
}
14:13:27.633 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:13:27.634 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.18.148.65"
port: 44837
}
bound_http_address {
host: "127.18.148.65"
port: 44653
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.18.148.66"
port: 38233
}
bound_http_address {
host: "127.18.148.66"
port: 44137
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.18.148.67"
port: 45649
}
bound_http_address {
host: "127.18.148.67"
port: 38005
}
}
}
14:13:27.634 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:13:27.636 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:13:27.637 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:27.638 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.148.126:46297(127.18.148.126:46297)] connecting to peer
14:13:27.639 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.148.125:40291(127.18.148.125:40291)] connecting to peer
14:13:27.639 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.148.124:37047(127.18.148.124:37047)] Successfully connected to peer
14:13:27.640 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.148.125:40291(127.18.148.125:40291)] Successfully connected to peer
14:13:27.642 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.148.126:46297(127.18.148.126:46297)] Successfully connected to peer
14:13:27.643 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.148.124
14:13:27.644 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.148.126
14:13:27.660 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.148.125
14:13:27.660 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7d5940f3, L:/127.0.0.1:36602 - R:/127.18.148.126:46297] using SASL/PLAIN
14:13:27.664 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0665c201, L:/127.0.0.1:43202 - R:/127.18.148.124:37047] using SASL/PLAIN
14:13:27.667 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:13:27.667 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=30), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [29ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, deferred=null) after lookup
14:13:27.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.669960 19070 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36602:
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:27.673805 19070 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:13:27.676 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x952a88fb, L:/127.0.0.1:33966 - R:/127.18.148.125:40291] using SASL/PLAIN
14:13:27.708 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table d6645c11bb1f4fd7a02eda5037b3db1e
14:13:27.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.754069 19735 tablet_service.cc:1468] Processing CreateTablet for tablet 2c486a200bdc4d68934444a51d47a683 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:13:27.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.755615 19434 tablet_service.cc:1468] Processing CreateTablet for tablet 2c486a200bdc4d68934444a51d47a683 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:13:27.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.754698 19736 tablet_service.cc:1468] Processing CreateTablet for tablet 1a0d3248fa9f4871a59d524ff7685c60 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:13:27.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.757526 19434 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2c486a200bdc4d68934444a51d47a683. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.757485 19735 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2c486a200bdc4d68934444a51d47a683. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.758764 19736 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1a0d3248fa9f4871a59d524ff7685c60. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.759371 19587 tablet_service.cc:1468] Processing CreateTablet for tablet 1a0d3248fa9f4871a59d524ff7685c60 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:13:27.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.761456 19587 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1a0d3248fa9f4871a59d524ff7685c60. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:27.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.761286 19586 tablet_service.cc:1468] Processing CreateTablet for tablet 2c486a200bdc4d68934444a51d47a683 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
14:13:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.761653 19435 tablet_service.cc:1468] Processing CreateTablet for tablet 1a0d3248fa9f4871a59d524ff7685c60 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
14:13:27.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.762578 19586 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2c486a200bdc4d68934444a51d47a683. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:27.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.762889 19435 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1a0d3248fa9f4871a59d524ff7685c60. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.806500 19827 tablet_bootstrap.cc:492] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09: Bootstrap starting.
14:13:27.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.809867 19829 tablet_bootstrap.cc:492] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b: Bootstrap starting.
14:13:27.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.812474 19828 tablet_bootstrap.cc:492] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec: Bootstrap starting.
14:13:27.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.821367 19827 tablet_bootstrap.cc:654] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09: Neither blocks nor log segments found. Creating new log.
14:13:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.822642 19829 tablet_bootstrap.cc:654] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b: Neither blocks nor log segments found. Creating new log.
14:13:27.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.823432 19828 tablet_bootstrap.cc:654] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec: Neither blocks nor log segments found. Creating new log.
14:13:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.824332 19827 log.cc:826] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09: Log is configured to *not* fsync() on all Append() calls
14:13:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.825466 19829 log.cc:826] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b: Log is configured to *not* fsync() on all Append() calls
14:13:27.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.826122 19828 log.cc:826] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec: Log is configured to *not* fsync() on all Append() calls
14:13:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.841526 19828 tablet_bootstrap.cc:492] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec: No bootstrap required, opened a new log
14:13:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.841671 19829 tablet_bootstrap.cc:492] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b: No bootstrap required, opened a new log
14:13:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.842212 19829 ts_tablet_manager.cc:1397] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b: Time spent bootstrapping tablet: real 0.033s user 0.019s sys 0.010s
14:13:27.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.842232 19828 ts_tablet_manager.cc:1397] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec: Time spent bootstrapping tablet: real 0.030s user 0.015s sys 0.012s
14:13:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.844386 19827 tablet_bootstrap.cc:492] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09: No bootstrap required, opened a new log
14:13:27.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.845070 19827 ts_tablet_manager.cc:1397] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09: Time spent bootstrapping tablet: real 0.039s user 0.011s sys 0.010s
14:13:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.866986 19829 raft_consensus.cc:357] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.867813 19829 raft_consensus.cc:383] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:27.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.868129 19829 raft_consensus.cc:738] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef36fff320fc4f2d9f7c375323b74a1b, State: Initialized, Role: FOLLOWER
14:13:27.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.869030 19829 consensus_queue.cc:260] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.869884 19828 raft_consensus.cc:357] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.870815 19828 raft_consensus.cc:383] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:27.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.871114 19828 raft_consensus.cc:738] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a31e618d3e354bdb84eca1dd32db33ec, State: Initialized, Role: FOLLOWER
14:13:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.872633 19827 raft_consensus.cc:357] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.873543 19827 raft_consensus.cc:383] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.873708 19829 ts_tablet_manager.cc:1428] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b: Time spent starting tablet: real 0.031s user 0.024s sys 0.006s
14:13:27.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.873831 19827 raft_consensus.cc:738] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f12ed04657c44a34b657c4b771901c09, State: Initialized, Role: FOLLOWER
14:13:27.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.872189 19828 consensus_queue.cc:260] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.874806 19827 consensus_queue.cc:260] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.876943 19829 tablet_bootstrap.cc:492] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b: Bootstrap starting.
14:13:27.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.883731 19829 tablet_bootstrap.cc:654] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b: Neither blocks nor log segments found. Creating new log.
14:13:27.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.896006 19827 ts_tablet_manager.cc:1428] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09: Time spent starting tablet: real 0.051s user 0.029s sys 0.008s
14:13:27.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.896416 19803 heartbeater.cc:499] Master 127.18.148.126:46297 was elected leader, sending a full tablet report...
14:13:27.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.897076 19827 tablet_bootstrap.cc:492] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09: Bootstrap starting.
14:13:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.906924 19828 ts_tablet_manager.cc:1428] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec: Time spent starting tablet: real 0.064s user 0.030s sys 0.005s
14:13:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.908161 19828 tablet_bootstrap.cc:492] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec: Bootstrap starting.
14:13:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.910043 19835 raft_consensus.cc:491] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.910605 19835 raft_consensus.cc:513] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:27.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.919648 19828 tablet_bootstrap.cc:654] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec: Neither blocks nor log segments found. Creating new log.
14:13:27.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.920224 19827 tablet_bootstrap.cc:654] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09: Neither blocks nor log segments found. Creating new log.
14:13:27.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.931564 19835 leader_election.cc:290] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837), f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:27.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:27.933058 19508 tablet.cc:2378] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:13:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.937970 19827 tablet_bootstrap.cc:492] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09: No bootstrap required, opened a new log
14:13:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.938570 19827 ts_tablet_manager.cc:1397] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09: Time spent bootstrapping tablet: real 0.042s user 0.011s sys 0.007s
14:13:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.941111 19827 raft_consensus.cc:357] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.941844 19827 raft_consensus.cc:383] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.942342 19827 raft_consensus.cc:738] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f12ed04657c44a34b657c4b771901c09, State: Initialized, Role: FOLLOWER
14:13:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.943150 19827 consensus_queue.cc:260] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:27.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.952966 19827 ts_tablet_manager.cc:1428] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09: Time spent starting tablet: real 0.014s user 0.007s sys 0.000s
14:13:27.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.954294 19756 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2c486a200bdc4d68934444a51d47a683" candidate_uuid: "a31e618d3e354bdb84eca1dd32db33ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f12ed04657c44a34b657c4b771901c09" is_pre_election: true
14:13:27.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.955128 19756 raft_consensus.cc:2466] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a31e618d3e354bdb84eca1dd32db33ec in term 0.
14:13:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.956607 19528 leader_election.cc:304] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a31e618d3e354bdb84eca1dd32db33ec, f12ed04657c44a34b657c4b771901c09; no voters:
14:13:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.957602 19835 raft_consensus.cc:2802] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Leader pre-election won for term 1
14:13:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.957980 19835 raft_consensus.cc:491] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:13:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.958299 19835 raft_consensus.cc:3058] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Advancing to term 1
14:13:27.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.961812 19828 tablet_bootstrap.cc:492] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec: No bootstrap required, opened a new log
14:13:27.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.962275 19828 ts_tablet_manager.cc:1397] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec: Time spent bootstrapping tablet: real 0.054s user 0.008s sys 0.004s
14:13:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.965008 19828 raft_consensus.cc:357] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.965736 19828 raft_consensus.cc:383] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.966019 19828 raft_consensus.cc:738] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a31e618d3e354bdb84eca1dd32db33ec, State: Initialized, Role: FOLLOWER
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.965659 19835 raft_consensus.cc:513] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.966675 19828 consensus_queue.cc:260] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.969070 19835 leader_election.cc:290] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [CANDIDATE]: Term 1 election: Requested vote from peers ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837), f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.970155 19828 ts_tablet_manager.cc:1428] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec: Time spent starting tablet: real 0.008s user 0.004s sys 0.004s
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.970454 19756 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2c486a200bdc4d68934444a51d47a683" candidate_uuid: "a31e618d3e354bdb84eca1dd32db33ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f12ed04657c44a34b657c4b771901c09"
14:13:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.971047 19756 raft_consensus.cc:3058] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Advancing to term 1
14:13:27.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.972602 19833 raft_consensus.cc:491] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.973232 19833 raft_consensus.cc:513] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.976442 19833 leader_election.cc:290] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649), a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233)
14:13:27.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.978129 19756 raft_consensus.cc:2466] T 2c486a200bdc4d68934444a51d47a683 P f12ed04657c44a34b657c4b771901c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a31e618d3e354bdb84eca1dd32db33ec in term 1.
14:13:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.980436 19528 leader_election.cc:304] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a31e618d3e354bdb84eca1dd32db33ec, f12ed04657c44a34b657c4b771901c09; no voters:
14:13:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.988585 19835 raft_consensus.cc:2802] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 1 FOLLOWER]: Leader election won for term 1
14:13:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.992594 19835 raft_consensus.cc:695] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [term 1 LEADER]: Becoming Leader. State: Replica: a31e618d3e354bdb84eca1dd32db33ec, State: Running, Role: LEADER
14:13:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:27.993933 19835 consensus_queue.cc:237] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.044602 19756 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1a0d3248fa9f4871a59d524ff7685c60" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f12ed04657c44a34b657c4b771901c09" is_pre_election: true
14:13:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.045258 19756 raft_consensus.cc:2466] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 0.
14:13:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.046706 19375 leader_election.cc:304] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ef36fff320fc4f2d9f7c375323b74a1b, f12ed04657c44a34b657c4b771901c09; no voters:
14:13:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.047667 19833 raft_consensus.cc:2802] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Leader pre-election won for term 1
14:13:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.048032 19833 raft_consensus.cc:491] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:13:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.048331 19833 raft_consensus.cc:3058] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Advancing to term 1
14:13:28.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.043922 19070 catalog_manager.cc:5582] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec reported cstate change: term changed from 0 to 1, leader changed from <none> to a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66). New cstate: current_term: 1 leader_uuid: "a31e618d3e354bdb84eca1dd32db33ec" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } health_report { overall_health: UNKNOWN } } }
14:13:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.053000 19607 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1a0d3248fa9f4871a59d524ff7685c60" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a31e618d3e354bdb84eca1dd32db33ec" is_pre_election: true
14:13:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.053817 19607 raft_consensus.cc:2466] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 0.
14:13:28.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.055272 19833 raft_consensus.cc:513] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.057210 19833 leader_election.cc:290] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 election: Requested vote from peers f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649), a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233)
14:13:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.058173 19607 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1a0d3248fa9f4871a59d524ff7685c60" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a31e618d3e354bdb84eca1dd32db33ec"
14:13:28.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.058734 19607 raft_consensus.cc:3058] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Advancing to term 1
14:13:28.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.060050 19829 tablet_bootstrap.cc:492] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b: No bootstrap required, opened a new log
14:13:28.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.060492 19829 ts_tablet_manager.cc:1397] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b: Time spent bootstrapping tablet: real 0.184s user 0.070s sys 0.092s
14:13:28.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.063228 19829 raft_consensus.cc:357] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.063920 19829 raft_consensus.cc:383] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.064218 19829 raft_consensus.cc:738] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef36fff320fc4f2d9f7c375323b74a1b, State: Initialized, Role: FOLLOWER
14:13:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.065179 19829 consensus_queue.cc:260] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.065943 19607 raft_consensus.cc:2466] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 1.
14:13:28.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.067802 19829 ts_tablet_manager.cc:1428] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b: Time spent starting tablet: real 0.007s user 0.002s sys 0.004s
14:13:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.070294 19375 leader_election.cc:304] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a31e618d3e354bdb84eca1dd32db33ec, ef36fff320fc4f2d9f7c375323b74a1b; no voters:
14:13:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.071144 19833 raft_consensus.cc:2802] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 1 FOLLOWER]: Leader election won for term 1
14:13:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.073666 19756 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "1a0d3248fa9f4871a59d524ff7685c60" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f12ed04657c44a34b657c4b771901c09"
14:13:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.074239 19756 raft_consensus.cc:3058] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Advancing to term 1
14:13:28.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.074565 19455 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2c486a200bdc4d68934444a51d47a683" candidate_uuid: "a31e618d3e354bdb84eca1dd32db33ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" is_pre_election: true
14:13:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.075579 19455 raft_consensus.cc:2466] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a31e618d3e354bdb84eca1dd32db33ec in term 0.
14:13:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.077381 19454 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "2c486a200bdc4d68934444a51d47a683" candidate_uuid: "a31e618d3e354bdb84eca1dd32db33ec" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef36fff320fc4f2d9f7c375323b74a1b"
14:13:28.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.078195 19454 raft_consensus.cc:3058] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Advancing to term 1
14:13:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:28.090914 19805 tablet.cc:2378] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:13:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.099040 19454 raft_consensus.cc:2466] T 2c486a200bdc4d68934444a51d47a683 P ef36fff320fc4f2d9f7c375323b74a1b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a31e618d3e354bdb84eca1dd32db33ec in term 1.
14:13:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.116413 19756 raft_consensus.cc:2466] T 1a0d3248fa9f4871a59d524ff7685c60 P f12ed04657c44a34b657c4b771901c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 1.
14:13:28.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.142527 19833 raft_consensus.cc:695] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [term 1 LEADER]: Becoming Leader. State: Replica: ef36fff320fc4f2d9f7c375323b74a1b, State: Running, Role: LEADER
14:13:28.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.143671 19833 consensus_queue.cc:237] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } }
14:13:28.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:28.149722 19655 tablet.cc:2378] T 1a0d3248fa9f4871a59d524ff7685c60 P a31e618d3e354bdb84eca1dd32db33ec: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:13:28.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.158577 19068 catalog_manager.cc:5582] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b reported cstate change: term changed from 0 to 1, leader changed from <none> to ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65). New cstate: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } health_report { overall_health: UNKNOWN } } }
14:13:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.578756 19847 consensus_queue.cc:1035] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:13:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.605835 19850 consensus_queue.cc:1035] T 2c486a200bdc4d68934444a51d47a683 P a31e618d3e354bdb84eca1dd32db33ec [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:13:28.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.635910 19833 consensus_queue.cc:1035] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [LEADER]: Connected to new peer: Peer: permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:13:28.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.678316 19833 consensus_queue.cc:1035] T 1a0d3248fa9f4871a59d524ff7685c60 P ef36fff320fc4f2d9f7c375323b74a1b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.739090 19068 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36606:
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:13:28.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:13:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:28.741452 19068 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:13:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.777704 19435 tablet_service.cc:1468] Processing CreateTablet for tablet c0946b5276f34835bddaad421b264c14 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9d07879e272645ad8e15282420750e70]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:13:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.779422 19435 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c0946b5276f34835bddaad421b264c14. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.779281 19587 tablet_service.cc:1468] Processing CreateTablet for tablet c0946b5276f34835bddaad421b264c14 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9d07879e272645ad8e15282420750e70]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:13:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.780964 19587 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c0946b5276f34835bddaad421b264c14. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.780592 19735 tablet_service.cc:1468] Processing CreateTablet for tablet c0946b5276f34835bddaad421b264c14 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=9d07879e272645ad8e15282420750e70]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
14:13:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.781771 19735 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c0946b5276f34835bddaad421b264c14. 1 dirs total, 0 dirs full, 0 dirs failed
14:13:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.798375 19880 tablet_bootstrap.cc:492] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: Bootstrap starting.
14:13:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.800231 19881 tablet_bootstrap.cc:492] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: Bootstrap starting.
14:13:28.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.801785 19879 tablet_bootstrap.cc:492] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: Bootstrap starting.
14:13:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.806420 19880 tablet_bootstrap.cc:654] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: Neither blocks nor log segments found. Creating new log.
14:13:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.806420 19881 tablet_bootstrap.cc:654] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: Neither blocks nor log segments found. Creating new log.
14:13:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.806514 19879 tablet_bootstrap.cc:654] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: Neither blocks nor log segments found. Creating new log.
14:13:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.815286 19879 tablet_bootstrap.cc:492] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: No bootstrap required, opened a new log
14:13:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.816136 19879 ts_tablet_manager.cc:1397] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: Time spent bootstrapping tablet: real 0.015s user 0.013s sys 0.000s
14:13:28.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.817858 19881 tablet_bootstrap.cc:492] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: No bootstrap required, opened a new log
14:13:28.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.818428 19881 ts_tablet_manager.cc:1397] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: Time spent bootstrapping tablet: real 0.019s user 0.008s sys 0.008s
14:13:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.821266 19881 raft_consensus.cc:357] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.822080 19881 raft_consensus.cc:383] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:28.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.822387 19881 raft_consensus.cc:738] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a31e618d3e354bdb84eca1dd32db33ec, State: Initialized, Role: FOLLOWER
14:13:28.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.823226 19881 consensus_queue.cc:260] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.825052 19879 raft_consensus.cc:357] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.825330 19847 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.825804 19879 raft_consensus.cc:383] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.826004 19847 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: This TxnStatusTablet replica's current role is: FOLLOWER
14:13:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.826105 19879 raft_consensus.cc:738] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f12ed04657c44a34b657c4b771901c09, State: Initialized, Role: FOLLOWER
14:13:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.826778 19881 ts_tablet_manager.cc:1428] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: Time spent starting tablet: real 0.008s user 0.008s sys 0.000s
14:13:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.826890 19879 consensus_queue.cc:260] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.828392 19848 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.829155 19848 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: This TxnStatusTablet replica's current role is: FOLLOWER
14:13:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.829224 19879 ts_tablet_manager.cc:1428] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: Time spent starting tablet: real 0.013s user 0.007s sys 0.000s
14:13:28.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.833003 19880 tablet_bootstrap.cc:492] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: No bootstrap required, opened a new log
14:13:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.833431 19880 ts_tablet_manager.cc:1397] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: Time spent bootstrapping tablet: real 0.035s user 0.015s sys 0.007s
14:13:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.836072 19880 raft_consensus.cc:357] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.836788 19880 raft_consensus.cc:383] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:13:28.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.837096 19880 raft_consensus.cc:738] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef36fff320fc4f2d9f7c375323b74a1b, State: Initialized, Role: FOLLOWER
14:13:28.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.837736 19880 consensus_queue.cc:260] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.840005 19880 ts_tablet_manager.cc:1428] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: Time spent starting tablet: real 0.006s user 0.004s sys 0.000s
14:13:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.840901 19833 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.841581 19833 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: This TxnStatusTablet replica's current role is: FOLLOWER
14:13:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.870600 19833 raft_consensus.cc:491] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:13:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.871006 19833 raft_consensus.cc:513] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.872536 19833 leader_election.cc:290] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233), f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.873502 19607 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c0946b5276f34835bddaad421b264c14" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a31e618d3e354bdb84eca1dd32db33ec" is_pre_election: true
14:13:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.873831 19756 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c0946b5276f34835bddaad421b264c14" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f12ed04657c44a34b657c4b771901c09" is_pre_election: true
14:13:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.874130 19607 raft_consensus.cc:2466] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 0.
14:13:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.874423 19756 raft_consensus.cc:2466] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 0.
14:13:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.875120 19375 leader_election.cc:304] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a31e618d3e354bdb84eca1dd32db33ec, ef36fff320fc4f2d9f7c375323b74a1b; no voters:
14:13:28.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.875762 19833 raft_consensus.cc:2802] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Leader pre-election won for term 1
14:13:28.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.876091 19833 raft_consensus.cc:491] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:13:28.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.876402 19833 raft_consensus.cc:3058] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 0 FOLLOWER]: Advancing to term 1
14:13:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.882547 19833 raft_consensus.cc:513] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.884271 19833 leader_election.cc:290] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 election: Requested vote from peers a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233), f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:28.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.885378 19756 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c0946b5276f34835bddaad421b264c14" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f12ed04657c44a34b657c4b771901c09"
14:13:28.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.885548 19607 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "c0946b5276f34835bddaad421b264c14" candidate_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a31e618d3e354bdb84eca1dd32db33ec"
14:13:28.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.885936 19756 raft_consensus.cc:3058] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [term 0 FOLLOWER]: Advancing to term 1
14:13:28.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.886044 19607 raft_consensus.cc:3058] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [term 0 FOLLOWER]: Advancing to term 1
14:13:28.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.892385 19607 raft_consensus.cc:2466] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 1.
14:13:28.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.892385 19756 raft_consensus.cc:2466] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef36fff320fc4f2d9f7c375323b74a1b in term 1.
14:13:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.893404 19375 leader_election.cc:304] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a31e618d3e354bdb84eca1dd32db33ec, ef36fff320fc4f2d9f7c375323b74a1b; no voters:
14:13:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.894030 19833 raft_consensus.cc:2802] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 1 FOLLOWER]: Leader election won for term 1
14:13:28.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.894474 19833 raft_consensus.cc:695] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [term 1 LEADER]: Becoming Leader. State: Replica: ef36fff320fc4f2d9f7c375323b74a1b, State: Running, Role: LEADER
14:13:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.895817 19833 consensus_queue.cc:237] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } }
14:13:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.902192 19870 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: TxnStatusTablet state changed. Reason: New leader ef36fff320fc4f2d9f7c375323b74a1b. Latest consensus state: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:28.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.902892 19870 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: This TxnStatusTablet replica's current role is: LEADER
14:13:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.906183 19886 txn_status_manager.cc:874] Waiting until node catch up with all replicated operations in previous term...
14:13:28.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.906744 19886 txn_status_manager.cc:930] Loading transaction status metadata into memory...
14:13:28.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.906809 19069 catalog_manager.cc:5582] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b reported cstate change: term changed from 0 to 1, leader changed from <none> to ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65). New cstate: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } health_report { overall_health: UNKNOWN } } }
14:13:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.971369 19756 raft_consensus.cc:1273] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09 [term 1 FOLLOWER]: Refusing update from remote peer ef36fff320fc4f2d9f7c375323b74a1b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:13:28.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.972680 19607 raft_consensus.cc:1273] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec [term 1 FOLLOWER]: Refusing update from remote peer ef36fff320fc4f2d9f7c375323b74a1b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:13:28.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.972790 19870 consensus_queue.cc:1035] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:13:28.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:28.973966 19833 consensus_queue.cc:1035] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b [LEADER]: Connected to new peer: Peer: permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:13:29.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.016804 19833 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:29.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.017961 19833 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: This TxnStatusTablet replica's current role is: LEADER
14:13:29.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.029028 19848 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: TxnStatusTablet state changed. Reason: New leader ef36fff320fc4f2d9f7c375323b74a1b. Latest consensus state: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:29.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.029760 19848 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: This TxnStatusTablet replica's current role is: FOLLOWER
14:13:29.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.030483 19847 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: TxnStatusTablet state changed. Reason: New leader ef36fff320fc4f2d9f7c375323b74a1b. Latest consensus state: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:29.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.031174 19847 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: This TxnStatusTablet replica's current role is: FOLLOWER
14:13:29.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.035559 19848 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:29.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.036178 19848 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P f12ed04657c44a34b657c4b771901c09: This TxnStatusTablet replica's current role is: FOLLOWER
14:13:29.040 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
14:13:29.049 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:894) Opened table d6645c11bb1f4fd7a02eda5037b3db1e
14:13:29.049 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.148.126:46297
14:13:29.050 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:13:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.052122 19870 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.052883 19870 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P ef36fff320fc4f2d9f7c375323b74a1b: This TxnStatusTablet replica's current role is: LEADER
14:13:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.053390 19025 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu with pid 19037
14:13:29.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.058101 19851 mvcc.cc:204] Tried to move back new op lower bound from 7170464190331961344 to 7170464190043570176. Current Snapshot: MvccSnapshot[applied={T|T < 7170464190331961344}]
14:13:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.063449 19847 tablet_replica.cc:440] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" member_type: VOTER last_known_addr { host: "127.18.148.65" port: 44837 } } peers { permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" member_type: VOTER last_known_addr { host: "127.18.148.66" port: 38233 } } peers { permanent_uuid: "f12ed04657c44a34b657c4b771901c09" member_type: VOTER last_known_addr { host: "127.18.148.67" port: 45649 } } }
14:13:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.064179 19847 tablet_replica.cc:443] T c0946b5276f34835bddaad421b264c14 P a31e618d3e354bdb84eca1dd32db33ec: This TxnStatusTablet replica's current role is: FOLLOWER
14:13:29.095 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.148.126:46297(127.18.148.126:46297)] handling channelInactive
14:13:29.095 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.126:46297(127.18.148.126:46297)] cleaning up while in state READY due to: connection closed
14:13:29.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.095921 19529 connection.cc:537] client connection to 127.18.148.126:46297 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:13:29.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.097860 19376 connection.cc:537] client connection to 127.18.148.126:46297 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:13:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.098841 19654 heartbeater.cc:646] Failed to heartbeat to 127.18.148.126:46297 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:13:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.099591 19505 heartbeater.cc:646] Failed to heartbeat to 127.18.148.126:46297 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:13:29.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.325685 19899 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 8cc3150135844c45b7060ca819868a7b)
14:13:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.326300 19899 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:29.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.328231 19899 leader_election.cc:290] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297), bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047)
14:13:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.329599 19254 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6536e6eac2a047b48c314cf1d86d1db2" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "bae83f02308f42caaaad05e349e4b26a" is_pre_election: true
14:13:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.330951 19140 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111) [suppressed 6 similar messages]
14:13:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.334600 19140 leader_election.cc:336] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111)
14:13:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.334954 19140 leader_election.cc:304] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6536e6eac2a047b48c314cf1d86d1db2; no voters: 8cc3150135844c45b7060ca819868a7b, bae83f02308f42caaaad05e349e4b26a
14:13:29.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.335610 19899 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:13:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.348532 19902 raft_consensus.cc:491] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 8cc3150135844c45b7060ca819868a7b)
14:13:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.349081 19902 raft_consensus.cc:513] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:29.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.351012 19902 leader_election.cc:290] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297), 6536e6eac2a047b48c314cf1d86d1db2 (127.18.148.125:40291)
14:13:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.352854 19161 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bae83f02308f42caaaad05e349e4b26a" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6536e6eac2a047b48c314cf1d86d1db2" is_pre_election: true
14:13:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.353610 19161 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bae83f02308f42caaaad05e349e4b26a in term 1.
14:13:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.354534 19229 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111)
14:13:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.354888 19228 leader_election.cc:304] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6536e6eac2a047b48c314cf1d86d1db2, bae83f02308f42caaaad05e349e4b26a; no voters:
14:13:29.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.355590 19902 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 1 FOLLOWER]: Leader pre-election won for term 2
14:13:29.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.356005 19902 raft_consensus.cc:491] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 1 FOLLOWER]: Starting leader election (detected failure of leader 8cc3150135844c45b7060ca819868a7b)
14:13:29.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.356374 19902 raft_consensus.cc:3058] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 1 FOLLOWER]: Advancing to term 2
14:13:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.359280 19229 leader_election.cc:336] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111)
14:13:29.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.363075 19902 raft_consensus.cc:513] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:29.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.365185 19902 leader_election.cc:290] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [CANDIDATE]: Term 2 election: Requested vote from peers 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297), 6536e6eac2a047b48c314cf1d86d1db2 (127.18.148.125:40291)
14:13:29.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.365835 19161 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bae83f02308f42caaaad05e349e4b26a" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "6536e6eac2a047b48c314cf1d86d1db2"
14:13:29.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.366390 19161 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 1 FOLLOWER]: Advancing to term 2
14:13:29.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.368033 19229 leader_election.cc:336] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111)
14:13:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.374541 19161 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bae83f02308f42caaaad05e349e4b26a in term 2.
14:13:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.375651 19228 leader_election.cc:304] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6536e6eac2a047b48c314cf1d86d1db2, bae83f02308f42caaaad05e349e4b26a; no voters: 8cc3150135844c45b7060ca819868a7b
14:13:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.376310 19902 raft_consensus.cc:2802] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 2 FOLLOWER]: Leader election won for term 2
14:13:29.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.378279 19902 raft_consensus.cc:695] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [term 2 LEADER]: Becoming Leader. State: Replica: bae83f02308f42caaaad05e349e4b26a, State: Running, Role: LEADER
14:13:29.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.379335 19902 consensus_queue.cc:237] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:29.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.382920 19905 sys_catalog.cc:455] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: SysCatalogTable state changed. Reason: New leader bae83f02308f42caaaad05e349e4b26a. Latest consensus state: current_term: 2 leader_uuid: "bae83f02308f42caaaad05e349e4b26a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:29.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.383700 19905 sys_catalog.cc:458] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: This master's current role is: LEADER
14:13:29.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.386659 19906 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:13:29.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.395596 19906 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=9d07879e272645ad8e15282420750e70]
14:13:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.397542 19906 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]
14:13:29.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.404803 19906 tablet_loader.cc:96] loaded metadata for tablet 1a0d3248fa9f4871a59d524ff7685c60 (table txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e])
14:13:29.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.406292 19906 tablet_loader.cc:96] loaded metadata for tablet 2c486a200bdc4d68934444a51d47a683 (table txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e])
14:13:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.407631 19906 tablet_loader.cc:96] loaded metadata for tablet c0946b5276f34835bddaad421b264c14 (table kudu_system.kudu_transactions [id=9d07879e272645ad8e15282420750e70])
14:13:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.409109 19906 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:13:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.413596 19906 catalog_manager.cc:1261] Loaded cluster ID: f5b6242f23d14f4e955c3f8ab9631ca3
14:13:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.413872 19906 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:13:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.418857 19906 catalog_manager.cc:1506] Loading token signing keys...
14:13:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.423532 19906 catalog_manager.cc:5966] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a: Loaded TSK: 0
14:13:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.425068 19906 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:13:29.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.485872 19161 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 2 FOLLOWER]: Refusing update from remote peer bae83f02308f42caaaad05e349e4b26a: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
14:13:29.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.487515 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:13:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.488193 19905 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [LEADER]: Connected to new peer: Peer: permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
14:13:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.491699 19899 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bae83f02308f42caaaad05e349e4b26a. Latest consensus state: current_term: 2 leader_uuid: "bae83f02308f42caaaad05e349e4b26a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.492509 19899 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: This master's current role is: FOLLOWER
14:13:29.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.498320 19905 sys_catalog.cc:455] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "bae83f02308f42caaaad05e349e4b26a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:29.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.499068 19905 sys_catalog.cc:458] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a [sys.catalog]: This master's current role is: LEADER
14:13:29.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.509699 19899 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "bae83f02308f42caaaad05e349e4b26a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:29.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.510332 19899 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: This master's current role is: FOLLOWER
14:13:29.558 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.126:46297(127.18.148.126:46297)] connecting to peer
14:13:29.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297
14:13:29.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.126:46297(127.18.148.126:46297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297
14:13:29.559 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.18.148.126:46297(127.18.148.126:46297) for tablet Kudu Master: Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297
14:13:29.577 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.126:46297(127.18.148.126:46297)] connecting to peer
14:13:29.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297
14:13:29.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.126:46297(127.18.148.126:46297)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297
14:13:29.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.126:46297: Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297
14:13:29.582 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:13:29.582 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=24), Traces: [0ms] sending RPC to server master-127.18.148.126:46297, [1ms] received response from server master-127.18.148.126:46297: Network error: Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [21ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: Network error: Failed to connect to peer master-127.18.148.126:46297(127.18.148.126:46297): Connection refused: /127.18.148.126:46297, [23ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: OK, deferred=Deferred@694112309(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/124904414@6d2b2761, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/1343146987@36e8c692)) after lookup
14:13:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:29.680130 19244 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43224:
14:13:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:13:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:13:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:13:29.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:29.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:13:29.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.684398 19244 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:13:29.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:29.974263 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:13:30.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:30.055604 19803 heartbeater.cc:646] Failed to heartbeat to 127.18.148.126:46297 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111)
14:13:30.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:30.470568 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:13:31.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.017403 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:13:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.049599 19801 heartbeater.cc:499] Master 127.18.148.124:37047 was elected leader, sending a full tablet report...
14:13:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.066502 19500 heartbeater.cc:499] Master 127.18.148.124:37047 was elected leader, sending a full tablet report...
14:13:31.108 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 1a0d3248fa9f4871a59d524ff7685c60 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
14:13:31.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 2c486a200bdc4d68934444a51d47a683 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
14:13:31.109 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=1a0d3248fa9f4871a59d524ff7685c60@[a31e618d3e354bdb84eca1dd32db33ec(127.18.148.66:38233),ef36fff320fc4f2d9f7c375323b74a1b(127.18.148.65:44837)[L],f12ed04657c44a34b657c4b771901c09(127.18.148.67:45649)]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=2c486a200bdc4d68934444a51d47a683@[a31e618d3e354bdb84eca1dd32db33ec(127.18.148.66:38233)[L],ef36fff320fc4f2d9f7c375323b74a1b(127.18.148.65:44837),f12ed04657c44a34b657c4b771901c09(127.18.148.67:45649)]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
14:13:31.109 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=4), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.18.148.124:37047, [3ms] Sub RPC GetTableLocations: received response from server master-127.18.148.124:37047: OK, deferred=null) after lookup
14:13:31.110 [DEBUG - kudu-nio-0] (Connection.java:748) [peer ef36fff320fc4f2d9f7c375323b74a1b(127.18.148.65:44837)] connecting to peer
14:13:31.111 [DEBUG - kudu-nio-1] (Connection.java:757) [peer ef36fff320fc4f2d9f7c375323b74a1b(127.18.148.65:44837)] Successfully connected to peer
14:13:31.126 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x352ca30b, L:/127.0.0.1:59974 - R:/127.18.148.65:44837] using TOKEN/null
14:13:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.134454 19388 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.18.148.126:46297)
14:13:31.210 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.148.126:46297
14:13:31.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:13:31.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.210736 19652 heartbeater.cc:499] Master 127.18.148.124:37047 was elected leader, sending a full tablet report...
14:13:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.211906 19025 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu
14:13:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/logs
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/info.pb
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.126:46297
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.126
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35611
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
14:13:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
14:13:31.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.531338 19947 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:13:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.531914 19947 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:13:31.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.532397 19947 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:13:31.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.566998 19947 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:13:31.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.567355 19947 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
14:13:31.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.567636 19947 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:13:31.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.567924 19947 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:13:31.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.568215 19947 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:13:31.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.568549 19947 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:13:31.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.568787 19947 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36457/jwks.json
14:13:31.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.605705 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.609056 19947 master_runner.cc:386] Master server non-default flags:
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.148.84:37513
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.148.126:46297,127.18.148.125:40291,127.18.148.124:37047
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.148.126:46297
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36457/jwks.json
14:13:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/info.pb
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.148.126
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35611
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/logs
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1550f239ea6e50bb8f84a51255fe1b8accb536aa
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 22 Jun 2025 13:43:21 UTC on 5fd53c4cbb9d
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6717
14:13:31.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:13:31.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.610791 19947 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:13:31.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.612525 19947 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:13:31.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.627761 19954 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:32.119012 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
14:13:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:32.657977 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
14:13:32.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:32.788240 19797 debug-util.cc:398] Leaking SignalData structure 0x7b08000d35c0 after lost signal to thread 19659
14:13:32.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:32.789608 19797 debug-util.cc:398] Leaking SignalData structure 0x7b080005cde0 after lost signal to thread 19800
14:13:32.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.629462 19956 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:31.630409 19947 server_base.cc:1048] running on GCE node
14:13:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:31.628121 19953 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:13:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:32.961954 19947 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:13:32.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:32.965358 19947 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:13:32.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:32.966831 19947 hybrid_clock.cc:648] HybridClock initialized: now 1750601612966690 us; error 146 us; skew 500 ppm
14:13:32.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:32.967772 19947 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:13:32.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:32.993934 19947 webserver.cc:469] Webserver started at http://127.18.148.126:35611/ using document root <none> and password file <none>
14:13:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:32.995245 19947 fs_manager.cc:362] Metadata directory not provided
14:13:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:32.995541 19947 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:13:33.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.007153 19947 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.008s sys 0.000s
14:13:33.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.013098 19966 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:13:33.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.014587 19947 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
14:13:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.015076 19947 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data,/tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8cc3150135844c45b7060ca819868a7b"
14:13:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-22 14:13:15 on dist-test-slave-p5xv"
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.018321 19947 fs_report.cc:389] FS layout report
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/wal
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/data
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:13:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:13:33.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.131242 19947 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:13:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.132898 19947 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:13:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.133374 19947 kserver.cc:163] Server-wide thread pool size limit: 3276
14:13:33.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:33.155386 19229 consensus_peers.cc:487] T 00000000000000000000000000000000 P bae83f02308f42caaaad05e349e4b26a -> Peer 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297): Couldn't send request to peer 8cc3150135844c45b7060ca819868a7b. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.126:46297: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
14:13:33.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.270325 19947 rpc_server.cc:307] RPC server started. Bound to: 127.18.148.126:46297
14:13:33.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.270433 20031 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.148.126:46297 every 8 connection(s)
14:13:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.274335 19947 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskNSAC30/test-tmp/mini-kudu-cluster7225113048713735051/master-0/data/info.pb
14:13:33.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.282425 19025 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu as pid 19947
14:13:33.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:33.283 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.148.124:37047
14:13:33.283 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:13:33.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.283859 19025 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu with pid 19202
14:13:33.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.290979 20032 sys_catalog.cc:263] Verifying existing consensus state
14:13:33.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.296344 20032 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Bootstrap starting.
14:13:33.327 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.148.124:37047(127.18.148.124:37047)] handling channelInactive
14:13:33.327 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state READY due to: connection closed
14:13:33.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:13:33.338 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.339 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.18.148.124:37047(127.18.148.124:37047) for tablet Kudu Master: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.343272 20032 log.cc:826] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Log is configured to *not* fsync() on all Append() calls
14:13:33.358 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.358 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.126:46297(127.18.148.126:46297)] connecting to peer
14:13:33.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.359 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.359 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.148.126:46297(127.18.148.126:46297)] Successfully connected to peer
14:13:33.379 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.18.148.126
14:13:33.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.387696 20032 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:13:33.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.389011 20032 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: Bootstrap complete.
14:13:33.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.419180 20032 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:33.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.422875 20032 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cc3150135844c45b7060ca819868a7b, State: Initialized, Role: FOLLOWER
14:13:33.424 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x0631de78, L:/127.0.0.1:51592 - R:/127.18.148.126:46297] using SASL/PLAIN
14:13:33.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.424477 20032 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:33.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.126:46297: Catalog manager is not initialized. State: Starting
14:13:33.429 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.438 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.126:46297: Catalog manager is not initialized. State: Starting
14:13:33.444 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.443921 20036 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:33.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.444903 20036 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: FOLLOWER
14:13:33.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.446214 20032 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: configured and running, proceeding with master startup.
14:13:33.458 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.459 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.461 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.126:46297: Catalog manager is not initialized. State: Starting
14:13:33.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.478 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.479 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.479 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.480 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.484 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.495339 20047 catalog_manager.cc:1261] Loaded cluster ID: f5b6242f23d14f4e955c3f8ab9631ca3
14:13:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.495739 20047 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: loading cluster ID for follower catalog manager: success
14:13:33.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.505738 20047 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: acquiring CA information for follower catalog manager: success
14:13:33.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.512691 20047 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:13:33.517 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.561348 20052 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader bae83f02308f42caaaad05e349e4b26a)
14:13:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.561934 20052 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.538746 19947 master_runner.cc:186] Error getting master registration for 127.18.148.124:37047: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111),
14:13:33.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.568269 19947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:13:33.577 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.581 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.581 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:33.591691 19141 leader_election.cc:336] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047): Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:33.593 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] sending RPC to server master-127.18.148.124:37047, [1ms] received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [21ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [25ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [91ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [91ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [105ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [105ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [121ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [122ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [124ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [141ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [145ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [181ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [183ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [243ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [248ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [253ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [254ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, deferred=Deferred@1868082748(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/124904414@6c7ff1a2 -> (continuation of Deferred@610207640 after retry RPC after error@1255482393) -> (continuation of Deferred@1616267825 after retry RPC after error@1941390712) -> (continuation of Deferred@789118801 after retry RPC after error@1277092677) -> (continuation of Deferred@508247861 after retry RPC after error@1469986049) -> (continuation of Deferred@1057270484 after retry RPC after error@71004858), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/1343146987@434e479f -> (continuation of Deferred@610207640 after retry RPC after error@1255482393) -> (continuation of Deferred@1616267825 after retry RPC after error@1941390712) -> (continuation of Deferred@789118801 after retry RPC after error@1277092677) -> (continuation of Deferred@508247861 after retry RPC after error@1469986049) -> (continuation of Deferred@1057270484 after retry RPC after error@71004858)))
14:13:33.593 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] sending RPC to server master-127.18.148.124:37047, [1ms] received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [21ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [25ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [91ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [91ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [105ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [105ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [121ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [122ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [124ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [125ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [139ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [141ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [145ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [145ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [146ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [181ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [183ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [183ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [243ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [248ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [253ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [254ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, deferred=Deferred@1868082748(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/124904414@6c7ff1a2 -> (continuation of Deferred@610207640 after retry RPC after error@1255482393) -> (continuation of Deferred@1616267825 after retry RPC after error@1941390712) -> (continuation of Deferred@789118801 after retry RPC after error@1277092677) -> (continuation of Deferred@508247861 after retry RPC after error@1469986049) -> (continuation of Deferred@1057270484 after retry RPC after error@71004858), errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/1343146987@434e479f -> (continuation of Deferred@610207640 after retry RPC after error@1255482393) -> (continuation of Deferred@1616267825 after retry RPC after error@1941390712) -> (continuation of Deferred@789118801 after retry RPC after error@1277092677) -> (continuation of Deferred@508247861 after retry RPC after error@1469986049) -> (continuation of Deferred@1057270484 after retry RPC after error@71004858)))
14:13:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.593032 20052 leader_election.cc:290] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297), bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047)
14:13:33.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.596047 19995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6536e6eac2a047b48c314cf1d86d1db2" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "8cc3150135844c45b7060ca819868a7b" is_pre_election: true
14:13:33.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.597056 19995 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6536e6eac2a047b48c314cf1d86d1db2 in term 1.
14:13:33.597 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.598519 19140 leader_election.cc:304] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6536e6eac2a047b48c314cf1d86d1db2, 8cc3150135844c45b7060ca819868a7b; no voters: bae83f02308f42caaaad05e349e4b26a
14:13:33.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.599910 20052 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:13:33.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.600339 20052 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 2 FOLLOWER]: Starting leader election (detected failure of leader bae83f02308f42caaaad05e349e4b26a)
14:13:33.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.601015 20052 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 2 FOLLOWER]: Advancing to term 3
14:13:33.605 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.605 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.605 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.610241 20052 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:33.616 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.617 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.620 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.620 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.620 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.620488 19995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6536e6eac2a047b48c314cf1d86d1db2" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "8cc3150135844c45b7060ca819868a7b"
14:13:33.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.621193 19995 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 1 FOLLOWER]: Advancing to term 3
14:13:33.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:33.630977 19141 leader_election.cc:336] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047): Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111)
14:13:33.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.641405 19995 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6536e6eac2a047b48c314cf1d86d1db2 in term 3.
14:13:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.642023 20052 leader_election.cc:290] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 3 election: Requested vote from peers 8cc3150135844c45b7060ca819868a7b (127.18.148.126:46297), bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047)
14:13:33.656 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.657 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.657289 19140 leader_election.cc:304] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6536e6eac2a047b48c314cf1d86d1db2, 8cc3150135844c45b7060ca819868a7b; no voters: bae83f02308f42caaaad05e349e4b26a
14:13:33.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.658810 20052 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 3 FOLLOWER]: Leader election won for term 3
14:13:33.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.662 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.673249 20052 raft_consensus.cc:695] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [term 3 LEADER]: Becoming Leader. State: Replica: 6536e6eac2a047b48c314cf1d86d1db2, State: Running, Role: LEADER
14:13:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.674629 20052 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } }
14:13:33.681 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.681895 20056 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6536e6eac2a047b48c314cf1d86d1db2. Latest consensus state: current_term: 3 leader_uuid: "6536e6eac2a047b48c314cf1d86d1db2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:33.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.682801 20056 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: This master's current role is: LEADER
14:13:33.697 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.702849 20058 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:13:33.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.712452 20058 catalog_manager.cc:671] Loaded metadata for table kudu_system.kudu_transactions [id=9d07879e272645ad8e15282420750e70]
14:13:33.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.714685 20058 catalog_manager.cc:671] Loaded metadata for table txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e]
14:13:33.717 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.718 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.719 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.719 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.730144 20058 tablet_loader.cc:96] loaded metadata for tablet 1a0d3248fa9f4871a59d524ff7685c60 (table txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e])
14:13:33.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.732852 20058 tablet_loader.cc:96] loaded metadata for tablet 2c486a200bdc4d68934444a51d47a683 (table txn_manager_fallback_rolling [id=d6645c11bb1f4fd7a02eda5037b3db1e])
14:13:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.735334 20058 tablet_loader.cc:96] loaded metadata for tablet c0946b5276f34835bddaad421b264c14 (table kudu_system.kudu_transactions [id=9d07879e272645ad8e15282420750e70])
14:13:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.737735 20058 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:13:33.739 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291; will retry
14:13:33.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.742551 20058 catalog_manager.cc:1261] Loaded cluster ID: f5b6242f23d14f4e955c3f8ab9631ca3
14:13:33.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.743001 20058 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:13:33.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.748420 20058 catalog_manager.cc:1506] Loading token signing keys...
14:13:33.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.753057 20058 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2: Loaded TSK: 0
14:13:33.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.754470 20058 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:13:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.774384 19995 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [term 3 FOLLOWER]: Refusing update from remote peer 6536e6eac2a047b48c314cf1d86d1db2: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
14:13:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.776551 20056 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.001s
14:13:33.778 [DEBUG - pool-40-thread-1] (Connection.java:748) [peer master-127.18.148.124:37047(127.18.148.124:37047)] connecting to peer
14:13:33.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.148.124:37047(127.18.148.124:37047)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.148.124:37047: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047
14:13:33.785 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:13:33.786 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=6, TimeoutTracker(timeout=250, elapsed=188), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [7ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [8ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [18ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [19ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [23ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [35ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [58ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [59ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [65ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [68ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [122ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [126ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [182ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [187ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, deferred=Deferred@1366458561(state=PENDING, result=null, callback=(continuation of Deferred@1714585476 after retry RPC after error@1032200317) -> (continuation of Deferred@1310359543 after retry RPC after error@187406973) -> (continuation of Deferred@2114984643 after retry RPC after error@1922652340) -> (continuation of Deferred@1868657879 after retry RPC after error@944698164) -> (continuation of Deferred@1182146812 after retry RPC after error@2123533438), errback=(continuation of Deferred@1714585476 after retry RPC after error@1032200317) -> (continuation of Deferred@1310359543 after retry RPC after error@187406973) -> (continuation of Deferred@2114984643 after retry RPC after error@1922652340) -> (continuation of Deferred@1868657879 after retry RPC after error@944698164) -> (continuation of Deferred@1182146812 after retry RPC after error@2123533438))) after lookup
14:13:33.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:33.795698 19141 consensus_peers.cc:487] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 -> Peer bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047): Couldn't send request to peer bae83f02308f42caaaad05e349e4b26a. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:13:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.805344 20036 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6536e6eac2a047b48c314cf1d86d1db2. Latest consensus state: current_term: 3 leader_uuid: "6536e6eac2a047b48c314cf1d86d1db2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.806371 20036 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: FOLLOWER
14:13:33.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.808171 20063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "6536e6eac2a047b48c314cf1d86d1db2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:33.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.808990 20063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: FOLLOWER
14:13:33.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.818759 20052 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "6536e6eac2a047b48c314cf1d86d1db2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:33.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.819543 20052 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 [sys.catalog]: This master's current role is: LEADER
14:13:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.829556 20063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "6536e6eac2a047b48c314cf1d86d1db2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cc3150135844c45b7060ca819868a7b" member_type: VOTER last_known_addr { host: "127.18.148.126" port: 46297 } } peers { permanent_uuid: "6536e6eac2a047b48c314cf1d86d1db2" member_type: VOTER last_known_addr { host: "127.18.148.125" port: 40291 } } peers { permanent_uuid: "bae83f02308f42caaaad05e349e4b26a" member_type: VOTER last_known_addr { host: "127.18.148.124" port: 37047 } } }
14:13:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.830374 20063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8cc3150135844c45b7060ca819868a7b [sys.catalog]: This master's current role is: FOLLOWER
14:13:33.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:33.849787 19185 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:33966 (request call id 14) took 61 ms (client timeout 61 ms). Trace:
14:13:33.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:33.850330 19185 rpcz_store.cc:269] 0622 14:13:33.788369 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:13:33.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:33.788653 (+ 284us) service_pool.cc:225] Handling call
14:13:33.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:33.849520 (+ 60867us) inbound_call.cc:173] Queueing success response
14:13:33.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"init.queue_time_us":3438,"thread_start_us":54755,"threads_started":5}
14:13:33.853 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [7ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [8ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [18ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [19ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [23ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [35ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [58ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [59ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [65ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [68ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [122ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [126ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [182ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [187ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [190ms] sending RPC to server master-127.18.148.125:40291, [210ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [255ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1366458561(state=PENDING, result=null, callback=(continuation of Deferred@1714585476 after retry RPC after error@1032200317) -> (continuation of Deferred@1310359543 after retry RPC after error@187406973) -> (continuation of Deferred@2114984643 after retry RPC after error@1922652340) -> (continuation of Deferred@1868657879 after retry RPC after error@944698164) -> (continuation of Deferred@1182146812 after retry RPC after error@2123533438) -> (continuation of Deferred@439457975 after retry RPC@1428754326), errback=(continuation of Deferred@1714585476 after retry RPC after error@1032200317) -> (continuation of Deferred@1310359543 after retry RPC after error@187406973) -> (continuation of Deferred@2114984643 after retry RPC after error@1922652340) -> (continuation of Deferred@1868657879 after retry RPC after error@944698164) -> (continuation of Deferred@1182146812 after retry RPC after error@2123533438) -> (continuation of Deferred@439457975 after retry RPC@1428754326)))
14:13:33.854 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=7, TimeoutTracker(timeout=250, elapsed=254), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [7ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [8ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [18ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [19ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [23ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [24ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [35ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [58ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [59ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [65ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [68ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [84ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [102ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [104ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [104ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [122ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [126ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [141ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.18.148.124:37047,127.18.148.126:46297,127.18.148.125:40291) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.124:37047, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.126:46297, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.148.125:40291, [182ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.124:37047: Network error: Failed to connect to peer master-127.18.148.124:37047(127.18.148.124:37047): Connection refused: /127.18.148.124:37047, [187ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.125:40291: OK, [190ms] sending RPC to server master-127.18.148.125:40291, [210ms] Sub RPC ConnectToMaster: received response from server master-127.18.148.126:46297: OK, [255ms] delaying RPC due to: Service unavailable: TxnManager is not yet initialized (error 0), deferred=Deferred@1366458561(state=PENDING, result=null, callback=(continuation of Deferred@1714585476 after retry RPC after error@1032200317) -> (continuation of Deferred@1310359543 after retry RPC after error@187406973) -> (continuation of Deferred@2114984643 after retry RPC after error@1922652340) -> (continuation of Deferred@1868657879 after retry RPC after error@944698164) -> (continuation of Deferred@1182146812 after retry RPC after error@2123533438) -> (continuation of Deferred@439457975 after retry RPC@1428754326), errback=(continuation of Deferred@1714585476 after retry RPC after error@1032200317) -> (continuation of Deferred@1310359543 after retry RPC after error@187406973) -> (continuation of Deferred@2114984643 after retry RPC after error@1922652340) -> (continuation of Deferred@1868657879 after retry RPC after error@944698164) -> (continuation of Deferred@1182146812 after retry RPC after error@2123533438) -> (continuation of Deferred@439457975 after retry RPC@1428754326)))
14:13:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.028364 19381 txn_status_manager.cc:1391] automatically aborted stale txn (ID 0) past 1.056s from last keepalive heartbeat (effective timeout is 1.000s)
14:13:34.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.040234 19389 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.18.148.124:37047)
14:13:34.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.051437 20069 tablet_replica.cc:1304] Aborted: operation has been aborted: cancelling pending write operations
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:33.957019 19155 catalog_manager.cc:2232] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51412:
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
14:13:34.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
14:13:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
14:13:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.062665 19155 catalog_manager.cc:6944] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
14:13:34.117 [DEBUG - pool-40-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=1, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.18.148.125:40291, deferred=Deferred@1040429260(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$119/124904414@643dd1dd, errback=org.apache.kudu.client.KuduTransaction$$Lambda$120/1343146987@7b18f14e))
14:13:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.144148 19505 heartbeater.cc:344] Connected to a master server at 127.18.148.126:46297
14:13:34.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.157295 20066 meta_cache.cc:302] tablet c0946b5276f34835bddaad421b264c14: replica ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837) has failed: Timed out: connection negotiation to 127.18.148.65:44837 for RPC CoordinateTransaction timed out after 0.021s (ON_OUTBOUND_QUEUE)
14:13:34.164 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=0, TimeoutTracker(timeout=0, elapsed=46), Traces: [0ms] received response from server master-127.18.148.125:40291: Timed out: Failed to write to server: (no server available): CoordinateTransaction(txn_status_tablet_id: "c0946b5276f34835bddaad421b264c14" op { type: KEEP_TXN_ALIVE txn_id: 0 user: "slave" }) passed its deadline: Timed out: connection negotiation to 127.18.148.65:44837 for RPC CoordinateTransaction timed out after 0.021s (ON_OUTBOUND_QUEUE) (error 0), deferred=null) with no deferred attached!
14:13:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.162118 19185 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:33966 (request call id 15) took 302 ms (client timeout 249 ms). Trace:
14:13:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.162493 19185 rpcz_store.cc:269] 0622 14:13:33.859642 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:13:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:33.859767 (+ 125us) service_pool.cc:225] Handling call
14:13:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0622 14:13:34.162034 (+302267us) inbound_call.cc:173] Queueing success response
14:13:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:13:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.156442 19985 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "ef36fff320fc4f2d9f7c375323b74a1b" instance_seqno: 1750601602766812) as {username='slave'} at 127.18.148.65:41671; Asking this server to re-register.
14:13:34.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.169517 19505 heartbeater.cc:461] Registering TS with master...
14:13:34.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.173815 19985 ts_manager.cc:194] Registered new tserver with Master: ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837)
14:13:34.175 [DEBUG - kudu-nio-0] (KuduTransaction.java:799) terminating keepalive task (txn ID 0) due to exception org.apache.kudu.client.NonRecoverableException: Failed to write to server: ef36fff320fc4f2d9f7c375323b74a1b (127.18.148.65:44837): transaction ID 0 is not available for further commits: state: ABORTED user: "slave" start_timestamp: 1750601608 last_transition_timestamp: 1750601614
14:13:34.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.181546 19803 heartbeater.cc:344] Connected to a master server at 127.18.148.126:46297
14:13:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.186403 19985 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "f12ed04657c44a34b657c4b771901c09" instance_seqno: 1750601607534235) as {username='slave'} at 127.18.148.67:43491; Asking this server to re-register.
14:13:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.187955 19803 heartbeater.cc:461] Registering TS with master...
14:13:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.192049 19984 ts_manager.cc:194] Registered new tserver with Master: f12ed04657c44a34b657c4b771901c09 (127.18.148.67:45649)
14:13:34.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.259367 19654 heartbeater.cc:344] Connected to a master server at 127.18.148.126:46297
14:13:34.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.264174 19985 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a31e618d3e354bdb84eca1dd32db33ec" instance_seqno: 1750601605346514) as {username='slave'} at 127.18.148.66:44555; Asking this server to re-register.
14:13:34.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.266217 19654 heartbeater.cc:461] Registering TS with master...
14:13:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:34.270195 19985 ts_manager.cc:194] Registered new tserver with Master: a31e618d3e354bdb84eca1dd32db33ec (127.18.148.66:38233)
14:13:34.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.319553 19141 consensus_peers.cc:487] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 -> Peer bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047): Couldn't send request to peer bae83f02308f42caaaad05e349e4b26a. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:13:34.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.417465 19141 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111) [suppressed 8 similar messages]
14:13:34.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:34.820132 19141 consensus_peers.cc:487] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 -> Peer bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047): Couldn't send request to peer bae83f02308f42caaaad05e349e4b26a. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
14:13:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.079859 19802 heartbeater.cc:499] Master 127.18.148.125:40291 was elected leader, sending a full tablet report...
14:13:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.114288 19501 heartbeater.cc:499] Master 127.18.148.125:40291 was elected leader, sending a full tablet report...
14:13:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.243865 19653 heartbeater.cc:499] Master 127.18.148.125:40291 was elected leader, sending a full tablet report...
14:13:35.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250622 14:13:35.297706 19141 consensus_peers.cc:487] T 00000000000000000000000000000000 P 6536e6eac2a047b48c314cf1d86d1db2 -> Peer bae83f02308f42caaaad05e349e4b26a (127.18.148.124:37047): Couldn't send request to peer bae83f02308f42caaaad05e349e4b26a. Status: Network error: Client connection negotiation failed: client connection to 127.18.148.124:37047: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
14:13:35.346 [DEBUG - main] (Connection.java:575) [peer master-127.18.148.124:37047(127.18.148.124:37047)] disconnecting while in state TERMINATED
14:13:35.346 [DEBUG - main] (Connection.java:575) [peer master-127.18.148.125:40291(127.18.148.125:40291)] disconnecting while in state READY
14:13:35.346 [DEBUG - main] (Connection.java:575) [peer master-127.18.148.126:46297(127.18.148.126:46297)] disconnecting while in state READY
14:13:35.347 [DEBUG - main] (Connection.java:575) [peer ef36fff320fc4f2d9f7c375323b74a1b(127.18.148.65:44837)] disconnecting while in state READY
14:13:35.348 [DEBUG - kudu-nio-1] (Connection.java:239) [peer ef36fff320fc4f2d9f7c375323b74a1b(127.18.148.65:44837)] handling channelInactive
14:13:35.348 [DEBUG - kudu-nio-1] (Connection.java:709) [peer ef36fff320fc4f2d9f7c375323b74a1b(127.18.148.65:44837)] cleaning up while in state READY due to: connection closed
14:13:35.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:13:35.352 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.148.125:40291(127.18.148.125:40291)] handling channelInactive
14:13:35.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.125:40291(127.18.148.125:40291)] cleaning up while in state READY due to: connection closed
14:13:35.353 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.148.126:46297(127.18.148.126:46297)] handling channelInactive
14:13:35.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.148.126:46297(127.18.148.126:46297)] cleaning up while in state READY due to: connection closed
14:13:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.353271 19025 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu with pid 19334
14:13:35.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.402491 19025 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu with pid 19504
14:13:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.433784 19025 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu with pid 19657
14:13:35.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.469460 19025 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu with pid 19947
14:13:35.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250622 14:13:35.508594 19025 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNSAC30/build/tsan/bin/kudu with pid 19117
14:13:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-22T14:13:36Z chronyd exiting
14:13:37.700 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
java.lang.AssertionError: Aborted: transaction was aborted
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)